Dockerfile: adding a dep here, since `dependencies` is not implemented

Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>

.github/workflows/python-lint.yml

@@ -9,14 +9,13 @@ jobs:
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v3
       with:
-        python-version: "3.10"
+        python-version: "3.11"
     - uses: isort/isort-action@master
       with:
         sortPaths: "./maubot"
     - uses: psf/black@stable
       with:
         src: "./maubot"
-        version: "22.1.0"
     - name: pre-commit
       run: |
         pip install pre-commit

.gitignore

@@ -7,7 +7,8 @@ pip-selfcheck.json
 *.pyc
 __pycache__
 
-*.db
+*.db*
+*.log
 /*.yaml
 !example-config.yaml
 !.pre-commit-config.yaml

.gitlab-ci-plugin.yml

@@ -0,0 +1,29 @@
+image: dock.mau.dev/maubot/maubot
+
+stages:
+- build
+
+variables:
+  PYTHONPATH: /opt/maubot
+
+build:
+  stage: build
+  except:
+  - tags
+  script:
+  - python3 -m maubot.cli build -o xyz.maubot.$CI_PROJECT_NAME-$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA.mbp
+  artifacts:
+    paths:
+    - "*.mbp"
+    expire_in: 365 days
+
+build tags:
+  stage: build
+  only:
+  - tags
+  script:
+  - python3 -m maubot.cli build -o xyz.maubot.$CI_PROJECT_NAME-$CI_COMMIT_TAG.mbp
+  artifacts:
+    paths:
+    - "*.mbp"
+    expire_in: never

.gitlab-ci.yml

@@ -10,7 +10,7 @@ default:
   - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
 
 build frontend:
-  image: node:16-alpine
+  image: node:18-alpine
   stage: build frontend
   before_script: []
   variables:

.pre-commit-config.yaml

@@ -1,23 +1,20 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
+    rev: v4.4.0
     hooks:
       - id: trailing-whitespace
         exclude_types: [markdown]
       - id: end-of-file-fixer
       - id: check-yaml
       - id: check-added-large-files
-  # TODO convert to use the upstream psf/black when
-  # https://github.com/psf/black/issues/2493 gets fixed
-  - repo: local
+  - repo: https://github.com/psf/black
+    rev: 23.1.0
     hooks:
       - id: black
-        name: black
-        entry: black --check
-        language: system
-        files: ^maubot/.*\.py$
+        language_version: python3
+        files: ^maubot/.*\.pyi?$
   - repo: https://github.com/PyCQA/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
       - id: isort
-        files: ^maubot/.*$
+        files: ^maubot/.*\.pyi?$

CHANGELOG.md

@@ -1,5 +1,45 @@
+# v0.4.2 (2023-09-20)
+
+* Updated Pillow to 10.0.1.
+* Updated Docker image to Alpine 3.18.
+* Added logging for errors for /whoami errors when adding new bot accounts.
+* Added support for using appservice tokens (including appservice encryption)
+  in standalone mode.
+
+# v0.4.1 (2023-03-15)
+
+* Added `in_thread` parameter to `evt.reply()` and `evt.respond()`.
+  * By default, responses will go to the thread if the command is in a thread.
+  * By setting the flag to `True` or `False`, the plugin can force the response
+    to either be or not be in a thread.
+* Fixed static files like the frontend app manifest not being served correctly.
+* Fixed `self.loader.meta` not being available to plugins in standalone mode.
+* Updated to mautrix-python v0.19.6.
+
+# v0.4.0 (2023-01-29)
+
+* Dropped support for using a custom maubot API base path.
+  * The public URL can still have a path prefix, e.g. when using a reverse
+    proxy. Both the web interface and `mbc` CLI tool should work fine with
+    custom prefixes.
+* Added `evt.redact()` as a shortcut for `self.client.redact(evt.room_id, evt.event_id)`.
+* Fixed `mbc logs` command not working on Python 3.8+.
+* Fixed saving plugin configs (broke in v0.3.0).
+* Fixed SSO login using the wrong API path (probably broke in v0.3.0).
+* Stopped using `cd` in the docker image's `mbc` wrapper to enable using
+  path-dependent commands like `mbc build` by mounting a directory.
+* Updated Docker image to Alpine 3.17.
+
+# v0.3.1 (2022-03-29)
+
+* Added encryption dependencies to standalone dockerfile.
+* Fixed running without encryption dependencies installed.
+* Removed unnecessary imports that broke on SQLAlchemy 1.4+.
+* Removed unused alembic dependency.
+
 # v0.3.0 (2022-03-28)
 
+* Dropped Python 3.7 support.
 * Switched main maubot database to asyncpg/aiosqlite.
   * Using the same SQLite database for crypto is now safe again.
 * Added support for asyncpg/aiosqlite for plugin databases.

Dockerfile

@@ -1,9 +1,9 @@
-FROM node:16 AS frontend-builder
+FROM node:18 AS frontend-builder
 
 COPY ./maubot/management/frontend /frontend
 RUN cd /frontend && yarn --prod && yarn build
 
-FROM alpine:3.15
+FROM alpine:3.18
 
 RUN apk add --no-cache \
         python3 py3-pip py3-setuptools py3-wheel \
@@ -21,11 +21,10 @@ RUN apk add --no-cache \
         py3-packaging \
         py3-markdown \
         py3-alembic \
-#        py3-cssselect \
+        py3-cssselect \
         py3-commonmark \
         py3-pygments \
         py3-tz \
-#        py3-tzlocal \
         py3-regex \
         py3-wcwidth \
         # encryption
@@ -35,13 +34,14 @@ RUN apk add --no-cache \
         py3-unpaddedbase64 \
         py3-future \
         # plugin deps
-        py3-pillow \
+        #py3-pillow \
         py3-magic \
         py3-feedparser \
         py3-dateutil \
-        py3-lxml
-#        py3-gitlab
-#        py3-semver@edge
+        py3-lxml \
+        py3-semver \
+  && python3 -m pip install pyfiglet \
+  && apk add --no-cache py3-pillow --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community
 # TODO remove pillow, magic, feedparser, lxml, gitlab and semver when maubot supports installing dependencies
 
 COPY requirements.txt /opt/maubot/requirements.txt
@@ -49,7 +49,7 @@ COPY optional-requirements.txt /opt/maubot/optional-requirements.txt
 WORKDIR /opt/maubot
 RUN apk add --virtual .build-deps python3-dev build-base git \
     && pip3 install -r requirements.txt -r optional-requirements.txt \
-        dateparser langdetect python-gitlab pyquery cchardet semver tzlocal cssselect \
+        dateparser langdetect python-gitlab pyquery tzlocal \
     && apk del .build-deps
 # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies
 

Dockerfile.ci

@@ -1,4 +1,4 @@
-FROM alpine:3.15
+FROM alpine:3.18
 
 RUN apk add --no-cache \
         python3 py3-pip py3-setuptools py3-wheel \
@@ -30,10 +30,11 @@ RUN apk add --no-cache \
         py3-unpaddedbase64 \
         py3-future \
         # plugin deps
-        py3-pillow \
+        #py3-pillow \
         py3-magic \
         py3-feedparser \
-        py3-lxml
+        py3-lxml \
+  && apk add --no-cache py3-pillow --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community
 #        py3-gitlab
 #        py3-semver
 # TODO remove pillow, magic, feedparser, lxml, gitlab and semver when maubot supports installing dependencies
@@ -43,7 +44,7 @@ COPY optional-requirements.txt /opt/maubot/optional-requirements.txt
 WORKDIR /opt/maubot
 RUN apk add --virtual .build-deps python3-dev build-base git \
     && pip3 install -r requirements.txt -r optional-requirements.txt \
-        dateparser langdetect python-gitlab pyquery cchardet semver tzlocal cssselect \
+        dateparser langdetect python-gitlab pyquery semver tzlocal cssselect \
     && apk del .build-deps
 # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies
 

README.md

@@ -22,56 +22,8 @@ All setup and usage instructions are located on
 Matrix room: [#maubot:maunium.net](https://matrix.to/#/#maubot:maunium.net)
 
 ## Plugins
-Open a pull request or join the Matrix room linked above to get your plugin listed here.
+A list of plugins can be found at [plugins.maubot.xyz](https://plugins.maubot.xyz/).
+
+To add your plugin to the list, send a pull request to <https://github.com/maubot/plugins.maubot.xyz>.
 
 The plugin wishlist lives at <https://github.com/maubot/plugin-wishlist/issues>.
-
-### Official plugins
-* [sed](https://github.com/maubot/sed) - A bot to do sed-like replacements.
-* [factorial](https://github.com/maubot/factorial) - A bot to calculate unexpected factorials.
-* [media](https://github.com/maubot/media) - A bot that replies with the MXC URI of images you send it.
-* [dice](https://github.com/maubot/dice) - A combined dice rolling and calculator bot.
-* [karma](https://github.com/maubot/karma) - A user karma tracker bot.
-* [xkcd](https://github.com/maubot/xkcd) - A bot to view xkcd comics.
-* [echo](https://github.com/maubot/echo) - A bot that echoes pings and other stuff.
-* [rss](https://github.com/maubot/rss) - A bot that posts RSS feed updates to Matrix.
-* [reminder](https://github.com/maubot/reminder) - A bot to remind you about things.
-* [translate](https://github.com/maubot/translate) - A bot to translate words.
-* [reactbot](https://github.com/maubot/reactbot) - A bot that responds to messages that match predefined rules.
-* [exec](https://github.com/maubot/exec) - A bot that executes code.
-* [commitstrip](https://github.com/maubot/commitstrip) - A bot to view CommitStrips.
-* [supportportal](https://github.com/maubot/supportportal) - A bot to manage customer support on Matrix.
-* †[gitlab](https://github.com/maubot/gitlab) - A GitLab client and webhook receiver.
-* [github](https://github.com/maubot/github) - A GitHub client and webhook receiver.
-* [tex](https://github.com/maubot/tex) - A bot that renders LaTeX.
-* [altalias](https://github.com/maubot/altalias) - A bot that lets users publish alternate aliases in rooms.
-* [satwcomic](https://github.com/maubot/satwcomic) - A bot to view SatWComics.
-* [songwhip](https://github.com/maubot/songwhip) - A bot to post Songwhip links.
-* [manhole](https://github.com/maubot/manhole) - A plugin that lets you access a Python shell inside maubot.
-
-### 3rd party plugins
-* [subreddit linkifier](https://github.com/TomCasavant/RedditMaubot) - A bot that condescendingly corrects a user when they enter an r/subreddit without providing a link to that subreddit
-* [giphy](https://github.com/TomCasavant/GiphyMaubot) - A bot that generates a gif (from giphy) given search terms
-* [trump](https://github.com/jeffcasavant/MaubotTrumpTweet) - A bot that generates a Trump tweet with the given content
-* [poll](https://github.com/TomCasavant/PollMaubot) - A bot that will create a simple poll for users in a room
-* [urban](https://github.com/dvdgsng/UrbanMaubot) - A bot that fetches definitions from [Urban Dictionary](https://www.urbandictionary.com/).
-* [twilio](https://github.com/jeffcasavant/MaubotTwilio) - Maubot-based SMS bridge
-* [tmdb](https://codeberg.org/lomion/tmdb-bot) - A bot that posts information about movies fetched from TheMovieDB.org.
-* [invite](https://github.com/williamkray/maubot-invite) - A bot to generate invitation tokens from [matrix-registration](https://github.com/ZerataX/matrix-registration)
-* [wolframalpha](https://github.com/ggogel/WolframAlphaMaubot) - A bot that allows requesting information from [WolframAlpha](https://www.wolframalpha.com/).
-* †[pingcheck](https://edugit.org/nik/maubot-pingcheck) - A bot to ping the echo bot and send rtt to Icinga passive check
-* [ticker](https://github.com/williamkray/maubot-ticker) - A bot to return financial data about a stock or cryptocurrency.
-* [weather](https://github.com/kellya/maubot-weather) - A bot to get the weather from wttr.in and return a single line of text for the location specified
-* †[youtube previewer](https://github.com/ggogel/YoutubePreviewMaubot) - A bot that responds to a YouTube link with the video title and thumbnail.
-* †[reddit previewer](https://github.com/ggogel/RedditPreviewMaubot) - A bot that responds to a link of a reddit post with the sub name and title. If available, uploads the image or video.
-* [pocket](https://github.com/jaywink/maubot-pocket) - A bot integrating with Pocket to fetch articles and archive them.
-* [alternatingcaps](https://github.com/rom4nik/maubot-alternatingcaps) - A bot repeating last message using aLtErNaTiNg cApS.
-* [metric](https://github.com/edwardsdean/maubot_metric_bot) - A bot that will reply to a message that contains imperial units and replace them with metric units.
-* [urlpreview](https://github.com/coffeebank/coffee-maubot/tree/master/urlpreview) - A bot that responds to links with a link preview embed, using Matrix API to fetch meta tags
-
-† Uses a synchronous library which can block the whole maubot process (e.g. requests instead of aiohttp)
-
-### Deprecated/unmaintained plugins
-* [jesaribot](https://github.com/maubot/jesaribot) - A simple bot that replies with an image when you say "jesari".
-  * Superseded by reactbot
-* [gitea](https://github.com/saces/maugitea) - A Gitea client and webhook receiver.

dev-requirements.txt

@@ -1,3 +1,3 @@
 pre-commit>=2.10.1,<3
 isort>=5.10.1,<6
-black==22.1.0
+black>=23,<24

docker/mbc.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
-cd /opt/maubot
+export PYTHONPATH=/opt/maubot
 python3 -m maubot.cli "$@"

docker/run.sh

@@ -13,7 +13,7 @@ function fixdefault {
 
 function fixconfig {
 	# Change relative default paths to absolute paths in /data
-	fixdefault '.database' 'sqlite:///maubot.db' 'sqlite:////data/maubot.db'
+	fixdefault '.database' 'sqlite:maubot.db' 'sqlite:/data/maubot.db'
 	fixdefault '.plugin_directories.upload' './plugins' '/data/plugins'
 	fixdefault '.plugin_directories.load[0]' './plugins' '/data/plugins'
 	fixdefault '.plugin_directories.trash' './trash' '/data/trash'
@@ -30,7 +30,6 @@ mkdir -p /var/log/maubot /data/plugins /data/trash /data/dbs
 
 if [ ! -f /data/config.yaml ]; then
 	cp example-config.yaml /data/config.yaml
-	# Apply some docker-specific adjustments to the config
 	echo "Config file not found. Example config copied to /data/config.yaml"
 	echo "Please modify the config file to your liking and restart the container."
 	fixperms
@@ -38,7 +37,6 @@ if [ ! -f /data/config.yaml ]; then
 	exit
 fi
 
-alembic -x config=/data/config.yaml upgrade head
 fixperms
 fixconfig
 if ls /data/plugins/*.db > /dev/null 2>&1; then

maubot/__main__.py

@@ -82,14 +82,17 @@ class Maubot(Program):
         )
         init_db(self.db)
 
-        if self.config["crypto_database"] == "default":
-            self.crypto_db = self.db
+        if PgCryptoStore:
+            if self.config["crypto_database"] == "default":
+                self.crypto_db = self.db
+            else:
+                self.crypto_db = Database.create(
+                    self.config["crypto_database"],
+                    upgrade_table=PgCryptoStore.upgrade_table,
+                    ignore_foreign_tables=self.args.ignore_foreign_tables,
+                )
         else:
-            self.crypto_db = Database.create(
-                self.config["crypto_database"],
-                upgrade_table=PgCryptoStore.upgrade_table,
-                ignore_foreign_tables=self.args.ignore_foreign_tables,
-            )
+            self.crypto_db = None
 
         if self.config["plugin_databases.postgres"] == "default":
             if self.db.scheme != Scheme.POSTGRES:
@@ -135,16 +138,17 @@ class Maubot(Program):
         ignore_unsupported = self.args.ignore_unsupported_database
         self.db.upgrade_table.allow_unsupported = ignore_unsupported
         self.state_store.upgrade_table.allow_unsupported = ignore_unsupported
-        PgCryptoStore.upgrade_table.allow_unsupported = ignore_unsupported
         try:
             await self.db.start()
             await self.state_store.upgrade_table.upgrade(self.db)
             if self.plugin_postgres_db and self.plugin_postgres_db is not self.db:
                 await self.plugin_postgres_db.start()
-            if self.crypto_db and self.crypto_db is not self.db:
-                await self.crypto_db.start()
-            else:
-                await PgCryptoStore.upgrade_table.upgrade(self.db)
+            if self.crypto_db:
+                PgCryptoStore.upgrade_table.allow_unsupported = ignore_unsupported
+                if self.crypto_db is not self.db:
+                    await self.crypto_db.start()
+                else:
+                    await PgCryptoStore.upgrade_table.upgrade(self.db)
         except DatabaseException as e:
             self.log.critical("Failed to initialize database", exc_info=e)
             if e.explanation:

maubot/__meta__.py

@@ -1 +1 @@
-__version__ = "0.3.0"
+__version__ = "0.4.2"

maubot/cli/commands/logs.py

@@ -38,13 +38,7 @@ def logs(server: str, tail: int) -> None:
     global history_count
     history_count = tail
     loop = asyncio.get_event_loop()
-    future = asyncio.create_task(view_logs(server, token), loop=loop)
-    try:
-        loop.run_until_complete(future)
-    except KeyboardInterrupt:
-        future.cancel()
-        loop.run_until_complete(future)
-        loop.close()
+    loop.run_until_complete(view_logs(server, token))
 
 
 def parsedate(entry: Obj) -> None:

maubot/cli/config.py

@@ -36,6 +36,7 @@ def get_default_server() -> tuple[str | None, str | None]:
         server = None
     if server is None:
         print(f"{Fore.RED}Default server not configured.{Fore.RESET}")
+        print(f"Perhaps you forgot to {Fore.CYAN}mbc login{Fore.RESET}?")
         return None, None
     return server, _get_token(server)
 

maubot/cli/util/spdx.py

@@ -36,10 +36,10 @@ def load() -> None:
 def get(id: str) -> dict[str, str]:
     if not spdx_list:
         load()
-    return spdx_list[id.lower()]
+    return spdx_list[id]
 
 
 def valid(id: str) -> bool:
     if not spdx_list:
         load()
-    return id.lower() in spdx_list
+    return id in spdx_list

maubot/client.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, AsyncGenerator, AsyncIterable, Awaitable, Callable, cast
+from typing import TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable, cast
 from collections import defaultdict
 import asyncio
 import logging
@@ -41,6 +41,7 @@ from mautrix.types import (
     SyncToken,
     UserID,
 )
+from mautrix.util import background_task
 from mautrix.util.async_getter_lock import async_getter_lock
 from mautrix.util.logging import TraceLogger
 
@@ -254,7 +255,7 @@ class Client(DBClient):
                 self.log.warning(
                     f"Failed to get /account/whoami, retrying in {(try_n + 1) * 10}s: {e}"
                 )
-                _ = asyncio.create_task(self.start(try_n + 1))
+                background_task.create(self.start(try_n + 1))
             return
         if whoami.user_id != self.id:
             self.log.error(f"User ID mismatch: expected {self.id}, but got {whoami.user_id}")

maubot/config.py

@@ -32,7 +32,11 @@ class Config(BaseFileConfig):
     def do_update(self, helper: ConfigUpdateHelper) -> None:
         base = helper.base
         copy = helper.copy
-        copy("database")
+
+        if "database" in self and self["database"].startswith("sqlite:///"):
+            helper.base["database"] = self["database"].replace("sqlite:///", "sqlite:")
+        else:
+            copy("database")
         copy("database_opts")
         if isinstance(self["crypto_database"], dict):
             if self["crypto_database.type"] == "postgres":
@@ -52,11 +56,9 @@ class Config(BaseFileConfig):
         copy("server.port")
         copy("server.public_url")
         copy("server.listen")
-        copy("server.base_path")
         copy("server.ui_base_path")
         copy("server.plugin_base_path")
         copy("server.override_resource_path")
-        copy("server.appservice_base_path")
         shared_secret = self["server.unshared_secret"]
         if shared_secret is None or shared_secret == "generate":
             base["server.unshared_secret"] = self._new_token()

maubot/db/__init__.py

@@ -1,7 +1,7 @@
 from mautrix.util.async_db import Database
 
 from .client import Client
-from .instance import Instance
+from .instance import DatabaseEngine, Instance
 from .upgrade import upgrade_table
 
 
@@ -10,4 +10,4 @@ def init(db: Database) -> None:
         table.db = db
 
 
-__all__ = ["upgrade_table", "init", "Client", "Instance"]
+__all__ = ["upgrade_table", "init", "Client", "Instance", "DatabaseEngine"]

maubot/db/instance.py

@@ -16,6 +16,7 @@
 from __future__ import annotations
 
 from typing import TYPE_CHECKING, ClassVar
+from enum import Enum
 
 from asyncpg import Record
 from attr import dataclass
@@ -26,6 +27,11 @@ from mautrix.util.async_db import Database
 fake_db = Database.create("") if TYPE_CHECKING else None
 
 
+class DatabaseEngine(Enum):
+    SQLITE = "sqlite"
+    POSTGRES = "postgres"
+
+
 @dataclass
 class Instance:
     db: ClassVar[Database] = fake_db
@@ -35,21 +41,31 @@ class Instance:
     enabled: bool
     primary_user: UserID
     config_str: str
+    database_engine: DatabaseEngine | None
+
+    @property
+    def database_engine_str(self) -> str | None:
+        return self.database_engine.value if self.database_engine else None
 
     @classmethod
     def _from_row(cls, row: Record | None) -> Instance | None:
         if row is None:
             return None
-        return cls(**row)
+        data = {**row}
+        db_engine = data.pop("database_engine", None)
+        return cls(**data, database_engine=DatabaseEngine(db_engine) if db_engine else None)
+
+    _columns = "id, type, enabled, primary_user, config, database_engine"
 
     @classmethod
     async def all(cls) -> list[Instance]:
-        rows = await cls.db.fetch("SELECT id, type, enabled, primary_user, config FROM instance")
+        q = f"SELECT {cls._columns} FROM instance"
+        rows = await cls.db.fetch(q)
         return [cls._from_row(row) for row in rows]
 
     @classmethod
     async def get(cls, id: str) -> Instance | None:
-        q = "SELECT id, type, enabled, primary_user, config FROM instance WHERE id=$1"
+        q = f"SELECT {cls._columns} FROM instance WHERE id=$1"
         return cls._from_row(await cls.db.fetchrow(q, id))
 
     async def update_id(self, new_id: str) -> None:
@@ -58,17 +74,27 @@ class Instance:
 
     @property
     def _values(self):
-        return self.id, self.type, self.enabled, self.primary_user, self.config_str
+        return (
+            self.id,
+            self.type,
+            self.enabled,
+            self.primary_user,
+            self.config_str,
+            self.database_engine_str,
+        )
 
     async def insert(self) -> None:
         q = (
-            "INSERT INTO instance (id, type, enabled, primary_user, config) "
-            "VALUES ($1, $2, $3, $4, $5)"
+            "INSERT INTO instance (id, type, enabled, primary_user, config, database_engine) "
+            "VALUES ($1, $2, $3, $4, $5, $6)"
         )
         await self.db.execute(q, *self._values)
 
     async def update(self) -> None:
-        q = "UPDATE instance SET type=$2, enabled=$3, primary_user=$4, config=$5 WHERE id=$1"
+        q = """
+        UPDATE instance SET type=$2, enabled=$3, primary_user=$4, config=$5, database_engine=$6
+        WHERE id=$1
+        """
         await self.db.execute(q, *self._values)
 
     async def delete(self) -> None:

maubot/db/upgrade/__init__.py

@@ -2,4 +2,4 @@ from mautrix.util.async_db import UpgradeTable
 
 upgrade_table = UpgradeTable()
 
-from . import v01_initial_revision
+from . import v01_initial_revision, v02_instance_database_engine

maubot/db/upgrade/v02_instance_database_engine.py

@@ -0,0 +1,25 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Store instance database engine")
+async def upgrade_v2(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE instance ADD COLUMN database_engine TEXT")

maubot/example-config.yaml

@@ -1,9 +1,9 @@
 # The full URI to the database. SQLite and Postgres are fully supported.
 # Other DBMSes supported by SQLAlchemy may or may not work.
 # Format examples:
-#   SQLite:   sqlite:///filename.db
+#   SQLite:   sqlite:filename.db
 #   Postgres: postgresql://username:password@hostname/dbname
-database: sqlite:///maubot.db
+database: sqlite:maubot.db
 
 # Separate database URL for the crypto database. "default" means use the same database as above.
 crypto_database: default
@@ -55,8 +55,6 @@ server:
     port: 29316
     # Public base URL where the server is visible.
     public_url: https://example.com
-    # The base management API path.
-    base_path: /_matrix/maubot/v1
     # The base path for the UI.
     ui_base_path: /_matrix/maubot
     # The base path for plugin endpoints. The instance ID will be appended directly.
@@ -64,8 +62,6 @@ server:
     # Override path from where to load UI resources.
     # Set to false to using pkg_resources to find the path.
     override_resource_path: false
-    # The base appservice API path. Use / for legacy appservice API and /_matrix/app/v1 for v1.
-    appservice_base_path: /_matrix/app/v1
     # The shared secret to sign API access tokens.
     # Set to "generate" to generate and save a new token at startup.
     unshared_secret: generate

maubot/handlers/command.py

@@ -72,7 +72,7 @@ class CommandHandler:
         self.__mb_must_consume_args__: bool = True
         self.__mb_arg_fallthrough__: bool = True
         self.__mb_event_handler__: bool = True
-        self.__mb_event_type__: EventType = EventType.ROOM_MESSAGE
+        self.__mb_event_types__: set[EventType] = {EventType.ROOM_MESSAGE}
         self.__mb_msgtypes__: Iterable[MessageType] = (MessageType.TEXT,)
         self.__bound_copies__: Dict[Any, CommandHandler] = {}
         self.__bound_instance__: Any = None
@@ -92,9 +92,10 @@ class CommandHandler:
                 "get_name",
                 "is_command_match",
                 "require_subcommand",
+                "must_consume_args",
                 "arg_fallthrough",
                 "event_handler",
-                "event_type",
+                "event_types",
                 "msgtypes",
             ]
             for key in keys:
@@ -314,7 +315,7 @@ def new(
         func.__mb_require_subcommand__ = require_subcommand
         func.__mb_arg_fallthrough__ = arg_fallthrough
         func.__mb_must_consume_args__ = must_consume_args
-        func.__mb_event_type__ = event_type
+        func.__mb_event_types__ = {event_type}
         if msgtypes:
             func.__mb_msgtypes__ = msgtypes
         return func

maubot/handlers/event.py

@@ -27,9 +27,12 @@ def on(var: EventType | InternalEventType | EventHandler) -> EventHandlerDecorat
     def decorator(func: EventHandler) -> EventHandler:
         func.__mb_event_handler__ = True
         if isinstance(var, (EventType, InternalEventType)):
-            func.__mb_event_type__ = var
+            if hasattr(func, "__mb_event_types__"):
+                func.__mb_event_types__.add(var)
+            else:
+                func.__mb_event_types__ = {var}
         else:
-            func.__mb_event_type__ = EventType.ALL
+            func.__mb_event_types__ = {EventType.ALL}
 
         return func
 

maubot/instance.py

@@ -28,13 +28,14 @@ from ruamel.yaml.comments import CommentedMap
 import sqlalchemy as sql
 
 from mautrix.types import UserID
+from mautrix.util import background_task
 from mautrix.util.async_db import Database, Scheme, UpgradeTable
 from mautrix.util.async_getter_lock import async_getter_lock
 from mautrix.util.config import BaseProxyConfig, RecursiveDict
 from mautrix.util.logging import TraceLogger
 
 from .client import Client
-from .db import Instance as DBInstance
+from .db import DatabaseEngine, Instance as DBInstance
 from .lib.plugin_db import ProxyPostgresDatabase
 from .loader import DatabaseType, PluginLoader, ZippedPluginLoader
 from .plugin_base import Plugin
@@ -71,10 +72,21 @@ class PluginInstance(DBInstance):
     started: bool
 
     def __init__(
-        self, id: str, type: str, enabled: bool, primary_user: UserID, config: str = ""
+        self,
+        id: str,
+        type: str,
+        enabled: bool,
+        primary_user: UserID,
+        config: str = "",
+        database_engine: DatabaseEngine | None = None,
     ) -> None:
         super().__init__(
-            id=id, type=type, enabled=bool(enabled), primary_user=primary_user, config_str=config
+            id=id,
+            type=type,
+            enabled=bool(enabled),
+            primary_user=primary_user,
+            config_str=config,
+            database_engine=database_engine,
         )
 
     def __hash__(self) -> int:
@@ -111,6 +123,8 @@ class PluginInstance(DBInstance):
             "database": (
                 self.inst_db is not None and self.maubot.config["api_features.instance_database"]
             ),
+            "database_interface": self.loader.meta.database_type_str if self.loader else "unknown",
+            "database_engine": self.database_engine_str,
         }
 
     def _introspect_sqlalchemy(self) -> dict:
@@ -263,18 +277,37 @@ class PluginInstance(DBInstance):
     def save_config(self, data: RecursiveDict[CommentedMap]) -> None:
         buf = io.StringIO()
         yaml.dump(data, buf)
-        self.config_str = buf.getvalue()
+        val = buf.getvalue()
+        if val != self.config_str:
+            self.config_str = val
+            self.log.debug("Creating background task to save updated config")
+            background_task.create(self.update())
 
     async def start_database(
         self, upgrade_table: UpgradeTable | None = None, actually_start: bool = True
     ) -> None:
         if self.loader.meta.database_type == DatabaseType.SQLALCHEMY:
+            if self.database_engine is None:
+                await self.update_db_engine(DatabaseEngine.SQLITE)
+            elif self.database_engine == DatabaseEngine.POSTGRES:
+                raise RuntimeError(
+                    "Instance database engine is marked as Postgres, but plugin uses legacy "
+                    "database interface, which doesn't support postgres."
+                )
             self.inst_db = sql.create_engine(f"sqlite:///{self._sqlite_db_path}")
         elif self.loader.meta.database_type == DatabaseType.ASYNCPG:
+            if self.database_engine is None:
+                if os.path.exists(self._sqlite_db_path) or not self.maubot.plugin_postgres_db:
+                    await self.update_db_engine(DatabaseEngine.SQLITE)
+                else:
+                    await self.update_db_engine(DatabaseEngine.POSTGRES)
             instance_db_log = db_log.getChild(self.id)
-            # TODO should there be a way to choose between SQLite and Postgres
-            #      for individual instances? Maybe checking the existence of the SQLite file.
-            if self.maubot.plugin_postgres_db:
+            if self.database_engine == DatabaseEngine.POSTGRES:
+                if not self.maubot.plugin_postgres_db:
+                    raise RuntimeError(
+                        "Instance database engine is marked as Postgres, but this maubot isn't "
+                        "configured to support Postgres for plugin databases"
+                    )
                 self.inst_db = ProxyPostgresDatabase(
                     pool=self.maubot.plugin_postgres_db,
                     instance_id=self.id,
@@ -284,7 +317,7 @@ class PluginInstance(DBInstance):
                 )
             else:
                 self.inst_db = Database.create(
-                    f"sqlite:///{self._sqlite_db_path}",
+                    f"sqlite:{self._sqlite_db_path}",
                     upgrade_table=upgrade_table,
                     log=instance_db_log,
                 )
@@ -334,7 +367,12 @@ class PluginInstance(DBInstance):
             self.log.debug("Disabling webapp after plugin meta reload")
             self.disable_webapp()
         if self.loader.meta.database:
-            await self.start_database(cls.get_db_upgrade_table())
+            try:
+                await self.start_database(cls.get_db_upgrade_table())
+            except Exception:
+                self.log.exception("Failed to start instance database")
+                await self.update_enabled(False)
+                return
         config_class = cls.get_config_class()
         if config_class:
             try:
@@ -455,6 +493,11 @@ class PluginInstance(DBInstance):
             self.enabled = enabled
             await self.update()
 
+    async def update_db_engine(self, db_engine: DatabaseEngine | None) -> None:
+        if db_engine is not None and db_engine != self.database_engine:
+            self.database_engine = db_engine
+            await self.update()
+
     @classmethod
     @async_getter_lock
     async def get(

maubot/lib/zipimport.py

@@ -323,6 +323,7 @@ _zip_searchorder = (
     (".py", False, False),
 )
 
+
 # Given a module name, return the potential file path in the
 # archive (without extension).
 def _get_module_path(self, fullname):
@@ -351,6 +352,7 @@ def _get_module_info(self, fullname):
 
 # implementation
 
+
 # _read_directory(archive) -> files dict (new reference)
 #
 # Given a path to a Zip archive, build a dict, mapping file names
@@ -524,6 +526,7 @@ cp437_table = (
 
 _importing_zlib = False
 
+
 # Return the zlib.decompress function object, or NULL if zlib couldn't
 # be imported. The function is cached when found, so subsequent calls
 # don't import zlib again.

maubot/loader/meta.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import List
+from typing import List, Optional
 
 from attr import dataclass
 from packaging.version import InvalidVersion, Version
@@ -63,3 +63,7 @@ class PluginMeta(SerializableAttrs):
     extra_files: List[str] = []
     dependencies: List[str] = []
     soft_dependencies: List[str] = []
+
+    @property
+    def database_type_str(self) -> Optional[str]:
+        return self.database_type.value if self.database else None

maubot/management/api/client.py

@@ -16,6 +16,7 @@
 from __future__ import annotations
 
 from json import JSONDecodeError
+import logging
 
 from aiohttp import web
 
@@ -27,6 +28,8 @@ from ...client import Client
 from .base import routes
 from .responses import resp
 
+log = logging.getLogger("maubot.server.client")
+
 
 @routes.get("/clients")
 async def get_clients(_: web.Request) -> web.Response:
@@ -54,11 +57,13 @@ async def _create_client(user_id: UserID | None, data: dict) -> web.Response:
     )
     try:
         whoami = await new_client.whoami()
-    except MatrixInvalidToken:
+    except MatrixInvalidToken as e:
         return resp.bad_client_access_token
     except MatrixRequestError:
+        log.warning(f"Failed to get whoami from {homeserver} for new client", exc_info=True)
         return resp.bad_client_access_details
     except MatrixConnectionError:
+        log.warning(f"Failed to connect to {homeserver} for new client", exc_info=True)
         return resp.bad_client_connection_details
     if user_id is None:
         existing_client = await Client.get(whoami.user_id)
@@ -90,8 +95,12 @@ async def _update_client(client: Client, data: dict, is_login: bool = False) ->
     except MatrixInvalidToken:
         return resp.bad_client_access_token
     except MatrixRequestError:
+        log.warning(
+            f"Failed to get whoami from homeserver to update client details", exc_info=True
+        )
         return resp.bad_client_access_details
     except MatrixConnectionError:
+        log.warning(f"Failed to connect to homeserver to update client details", exc_info=True)
         return resp.bad_client_connection_details
     except ValueError as e:
         str_err = str(e)

maubot/management/api/client_auth.py

@@ -184,11 +184,10 @@ async def _do_sso(req: AuthRequestInfo) -> web.Response:
     cfg = get_config()
     public_url = (
         URL(cfg["server.public_url"])
-        / cfg["server.base_path"].lstrip("/")
-        / "client/auth_external_sso/complete"
+        / "_matrix/maubot/v1/client/auth_external_sso/complete"
         / waiter_id
     )
-    sso_url = req.client.api.base_url.with_path(str(Path.login.sso.redirect)).with_query(
+    sso_url = req.client.api.base_url.with_path(str(Path.v3.login.sso.redirect)).with_query(
         {"redirectUrl": str(public_url)}
     )
     sso_waiters[waiter_id] = req, asyncio.get_running_loop().create_future()

maubot/management/api/instance_database.py

@@ -20,7 +20,6 @@ from datetime import datetime
 from aiohttp import web
 from asyncpg import PostgresError
 from sqlalchemy import asc, desc, engine, exc
-from sqlalchemy.engine.result import ResultProxy, RowProxy
 import aiosqlite
 
 from mautrix.util.async_db import Database
@@ -134,7 +133,7 @@ def _execute_query_sqlalchemy(
 ) -> web.Response:
     assert isinstance(instance.inst_db, engine.Engine)
     try:
-        res: ResultProxy = instance.inst_db.execute(sql_query)
+        res = instance.inst_db.execute(sql_query)
     except exc.IntegrityError as e:
         return resp.sql_integrity_error(e, sql_query)
     except exc.OperationalError as e:
@@ -144,7 +143,6 @@ def _execute_query_sqlalchemy(
         "query": str(sql_query),
     }
     if res.returns_rows:
-        row: RowProxy
         data["rows"] = [
             (
                 {key: check_type(value) for key, value in row.items()}

maubot/management/api/log.py

@@ -22,6 +22,8 @@ import logging
 
 from aiohttp import web, web_ws
 
+from mautrix.util import background_task
+
 from .auth import is_valid_token
 from .base import routes
 
@@ -142,7 +144,7 @@ async def log_websocket(request: web.Request) -> web.WebSocketResponse:
             await ws.close(code=4000)
             log.debug(f"Connection from {request.remote} terminated due to no authentication")
 
-    asyncio.create_task(close_if_not_authenticated())
+    background_task.create(close_if_not_authenticated())
 
     try:
         msg: web_ws.WSMessage

maubot/management/api/middleware.py

@@ -29,7 +29,7 @@ log = logging.getLogger("maubot.server")
 
 @web.middleware
 async def auth(request: web.Request, handler: Handler) -> web.Response:
-    subpath = request.path[len(get_config()["server.base_path"]) :]
+    subpath = request.path[len("/_matrix/maubot/v1") :]
     if (
         subpath.startswith("/auth/")
         or subpath.startswith("/client/auth_external_sso/complete/")

maubot/management/frontend/src/pages/Main.js

@@ -45,9 +45,8 @@ class Main extends Component {
             const resp = await fetch(process.env.PUBLIC_URL + "/paths.json", {
                 headers: { "Content-Type": "application/json" },
             })
-            const apiPathJson = await resp.json()
-            const apiPath = apiPathJson.api_path
-            api.setBasePath(`${apiPath}`)
+            const apiPaths = await resp.json()
+            api.setBasePath(apiPaths.api_path)
         } catch (err) {
             console.error("Failed to get API path:", err)
         }

maubot/management/frontend/src/pages/dashboard/Instance.js

@@ -43,7 +43,7 @@ class Instance extends BaseMainView {
     }
 
     get entryKeys() {
-        return ["id", "primary_user", "enabled", "started", "type", "config"]
+        return ["id", "primary_user", "enabled", "started", "type", "config", "database_engine"]
     }
 
     get initialState() {
@@ -54,6 +54,7 @@ class Instance extends BaseMainView {
             started: true,
             type: "",
             config: "",
+            database_engine: "",
 
             saving: false,
             deleting: false,

maubot/management/frontend/src/pages/dashboard/Log.js

@@ -41,7 +41,7 @@ class LogEntry extends PureComponent {
             const req = this.props.line.matrix_http_request
 
             return <>
-                {req.method} {req.path}
+                {req.method} {req.url || req.path}
                 <div className="content">
                     {Object.entries(req.content || {}).length > 0
                     && <JSONTree data={{ content: req.content }} hideRoot={true}/>}

maubot/matrix.py

@@ -24,6 +24,7 @@ import attr
 from mautrix.client import Client as MatrixClient, SyncStream
 from mautrix.errors import DecryptionError
 from mautrix.types import (
+    BaseMessageEventContentFuncs,
     EncryptedEvent,
     Event,
     EventID,
@@ -82,8 +83,31 @@ class MaubotMessageEvent(MessageEvent):
         markdown: bool = True,
         allow_html: bool = False,
         reply: bool | str = False,
+        in_thread: bool | None = None,
         edits: EventID | MessageEvent | None = None,
     ) -> EventID:
+        """
+        Respond to the message.
+
+        Args:
+            content: The content to respond with. If this is a string, it will be passed to
+                :func:`parse_formatted` with the markdown and allow_html flags.
+                Otherwise, the content is used as-is
+            event_type: The type of event to send.
+            markdown: When content is a string, should it be parsed as markdown?
+            allow_html: When content is a string, should it allow raw HTML?
+            reply: Should the response be sent as a reply to this event?
+            in_thread: Should the response be sent in a thread with this event?
+                By default (``None``), the response will be in a thread if this event is in a
+                thread. If set to ``False``, the response will never be in a thread. If set to
+                ``True``, the response will always be in a thread, creating one with this event as
+                the root if necessary.
+            edits: An event ID or MessageEvent to edit. If set, the reply and in_thread parameters
+                are ignored, as edits can't change the reply or thread status.
+
+        Returns:
+            The ID of the response event.
+        """
         if isinstance(content, str):
             content = TextMessageEventContent(msgtype=MessageType.NOTICE, body=content)
             if allow_html or markdown:
@@ -93,7 +117,19 @@ class MaubotMessageEvent(MessageEvent):
                 )
         if edits:
             content.set_edit(edits)
-        elif reply:
+        if (
+            not edits
+            and in_thread is not False
+            and (
+                in_thread
+                or (
+                    isinstance(self.content, BaseMessageEventContentFuncs)
+                    and self.content.get_thread_parent()
+                )
+            )
+        ):
+            content.set_thread_parent(self)
+        if reply and not edits:
             if reply != "force" and self.disable_reply:
                 content.body = f"{self.sender}: {content.body}"
                 fmt_body = content.formatted_body or escape(content.body).replace("\n", "<br>")
@@ -112,17 +148,71 @@ class MaubotMessageEvent(MessageEvent):
         event_type: EventType = EventType.ROOM_MESSAGE,
         markdown: bool = True,
         allow_html: bool = False,
+        in_thread: bool | None = None,
     ) -> Awaitable[EventID]:
+        """
+        Reply to the message. The parameters are the same as :meth:`respond`,
+        but ``reply`` is always ``True`` and ``edits`` is not supported.
+
+        Args:
+            content: The content to respond with. If this is a string, it will be passed to
+                :func:`parse_formatted` with the markdown and allow_html flags.
+                Otherwise, the content is used as-is
+            event_type: The type of event to send.
+            markdown: When content is a string, should it be parsed as markdown?
+            allow_html: When content is a string, should it allow raw HTML?
+            in_thread: Should the response be sent in a thread with this event?
+                By default (``None``), the response will be in a thread if this event is in a
+                thread. If set to ``False``, the response will never be in a thread. If set to
+                ``True``, the response will always be in a thread, creating one with this event as
+                the root if necessary.
+
+        Returns:
+            The ID of the response event.
+        """
         return self.respond(
-            content, event_type, markdown=markdown, reply=True, allow_html=allow_html
+            content,
+            event_type,
+            markdown=markdown,
+            reply=True,
+            in_thread=in_thread,
+            allow_html=allow_html,
         )
 
     def mark_read(self) -> Awaitable[None]:
+        """
+        Mark this event as read.
+        """
         return self.client.send_receipt(self.room_id, self.event_id, "m.read")
 
     def react(self, key: str) -> Awaitable[EventID]:
+        """
+        React to this event with the given key.
+
+        Args:
+            key: The key to react with. Often an unicode emoji.
+
+        Returns:
+            The ID of the reaction event.
+
+        Examples:
+            >>> evt: MaubotMessageEvent
+            >>> evt.react("🐈️")
+        """
         return self.client.react(self.room_id, self.event_id, key)
 
+    def redact(self, reason: str | None = None) -> Awaitable[EventID]:
+        """
+        Redact this event.
+
+        Args:
+            reason: Optionally, the reason for redacting the event.
+
+        Returns:
+            The ID of the redaction event.
+        """
+        return self.client.redact(self.room_id, self.event_id, reason=reason)
+
     def edit(
         self,
         content: str | MessageEventContent,
@@ -130,6 +220,21 @@ class MaubotMessageEvent(MessageEvent):
         markdown: bool = True,
         allow_html: bool = False,
     ) -> Awaitable[EventID]:
+        """
+        Edit this event. Note that other clients will only render the edit if it was sent by the
+        same user who's doing the editing.
+
+        Args:
+            content: The new content for the event. If this is a string, it will be passed to
+                :func:`parse_formatted` with the markdown and allow_html flags.
+                Otherwise, the content is used as-is.
+            event_type: The type of event to edit into.
+            markdown: When content is a string, should it be parsed as markdown?
+            allow_html: When content is a string, should it allow raw HTML?
+
+        Returns:
+            The ID of the edit event.
+        """
         return self.respond(
             content, event_type, markdown=markdown, edits=self, allow_html=allow_html
         )

maubot/plugin_base.py

@@ -76,8 +76,9 @@ class Plugin(ABC):
             val = getattr(obj, key)
             try:
                 if val.__mb_event_handler__:
-                    self._handlers_at_startup.append((val, val.__mb_event_type__))
-                    self.client.add_event_handler(val.__mb_event_type__, val)
+                    for event_type in val.__mb_event_types__:
+                        self._handlers_at_startup.append((val, event_type))
+                        self.client.add_event_handler(event_type, val)
             except AttributeError:
                 pass
             try:

maubot/server.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
+from typing import Callable
 import asyncio
 import json
 import logging
@@ -52,7 +53,7 @@ class MaubotServer:
         self.config = config
 
         self.setup_appservice()
-        self.app.add_subapp(config["server.base_path"], management_api)
+        self.app.add_subapp("/_matrix/maubot/v1", management_api)
         self.setup_instance_subapps()
         self.setup_management_ui()
 
@@ -93,7 +94,7 @@ class MaubotServer:
         self.app.router.register_resource(resource)
 
     def setup_appservice(self) -> None:
-        as_path = PathBuilder(self.config["server.appservice_base_path"])
+        as_path = PathBuilder("/_matrix/appservice/v1")
         self.add_route(Method.PUT, as_path.transactions, self.handle_transaction)
 
     def setup_management_ui(self) -> None:
@@ -127,6 +128,13 @@ class MaubotServer:
         )
         self.app.router.add_get(ui_base, ui_base_redirect)
 
+    @staticmethod
+    def _static_data(data: bytes, mime: str) -> Callable[[web.Request], web.Response]:
+        def fn(_: web.Request) -> web.Response:
+            return web.Response(body=data, content_type=mime)
+
+        return fn
+
     def setup_static_root_files(self, directory: str, ui_base: str) -> None:
         files = {
             "asset-manifest.json": "application/json",
@@ -136,20 +144,14 @@ class MaubotServer:
         for file, mime in files.items():
             with open(f"{directory}/{file}", "rb") as stream:
                 data = stream.read()
-            self.app.router.add_get(
-                f"{ui_base}/{file}", lambda _: web.Response(body=data, content_type=mime)
-            )
+            self.app.router.add_get(f"{ui_base}/{file}", self._static_data(data, mime))
 
-        # also set up a resource path for the public url path prefix config
-        # cut the prefix path from public_url
         public_url = self.config["server.public_url"]
-        base_path = self.config["server.base_path"]
         public_url_path = ""
         if public_url:
             public_url_path = URL(public_url).path.rstrip("/")
 
-        # assemble with base_path
-        api_path = f"{public_url_path}{base_path}"
+        api_path = f"{public_url_path}/_matrix/maubot/v1"
 
         path_prefix_response_body = json.dumps({"api_path": api_path.rstrip("/")})
         self.app.router.add_get(

maubot/standalone/Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/alpine:3.15
+FROM docker.io/alpine:3.18
 
 RUN apk add --no-cache \
       python3 py3-pip py3-setuptools py3-wheel \
@@ -13,15 +13,20 @@ RUN apk add --no-cache \
       py3-ruamel.yaml \
       py3-jinja2 \
       py3-packaging \
-      py3-markdown
+      py3-markdown \
+      py3-cffi \
+      py3-olm \
+      py3-pycryptodome \
+      py3-unpaddedbase64
 
 COPY requirements.txt /opt/maubot/requirements.txt
+COPY optional-requirements.txt /opt/maubot/optional-requirements.txt
 RUN cd /opt/maubot \
   && apk add --no-cache --virtual .build-deps \
       python3-dev \
       libffi-dev \
       build-base \
-  && pip3 install -r requirements.txt \
+  && pip3 install -r requirements.txt -r optional-requirements.txt \
   && apk del .build-deps
 
 COPY . /opt/maubot

maubot/standalone/__main__.py

@@ -30,7 +30,11 @@ from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap
 from yarl import URL
 
+from mautrix.appservice import AppServiceServerMixin
+from mautrix.client import SyncStream
 from mautrix.types import (
+    BaseMessageEventContentFuncs,
+    Event,
     EventType,
     Filter,
     Membership,
@@ -113,9 +117,12 @@ if "/" in meta.main_class:
 else:
     module = meta.modules[0]
     main_class = meta.main_class
+
+if args.meta != "maubot.yaml" and os.path.dirname(args.meta) != "":
+    sys.path.append(os.path.dirname(args.meta))
 bot_module = importlib.import_module(module)
 plugin: type[Plugin] = getattr(bot_module, main_class)
-loader = FileSystemLoader(os.path.dirname(args.meta))
+loader = FileSystemLoader(os.path.dirname(args.meta), meta)
 
 log.info(f"Initializing standalone {meta.id} v{meta.version} on maubot {__version__}")
 
@@ -131,6 +138,7 @@ user_id = config["user.credentials.id"]
 device_id = config["user.credentials.device_id"]
 homeserver = config["user.credentials.homeserver"]
 access_token = config["user.credentials.access_token"]
+appservice_listener = config["user.appservice"]
 
 crypto_store = state_store = None
 if device_id and not OlmMachine:
@@ -188,6 +196,10 @@ if meta.webapp:
     resource = PrefixResource(web_base_path)
     resource.add_route(hdrs.METH_ANY, _handle_plugin_request)
     web_app.router.register_resource(resource)
+elif appservice_listener:
+    web_app = web.Application()
+    web_runner = web.AppRunner(web_app, access_log_class=AccessLogger)
+    public_url = plugin_webapp = None
 else:
     web_app = web_runner = public_url = plugin_webapp = None
 
@@ -195,6 +207,31 @@ loop = asyncio.get_event_loop()
 
 client: MaubotMatrixClient | None = None
 bot: Plugin | None = None
+appservice: AppServiceServerMixin | None = None
+
+
+if appservice_listener:
+    assert web_app is not None, "web_app is always set when appservice_listener is set"
+    appservice = AppServiceServerMixin(
+        ephemeral_events=True,
+        encryption_events=True,
+        log=logging.getLogger("maubot.appservice"),
+        hs_token=config["user.hs_token"],
+    )
+    appservice.register_routes(web_app)
+
+    @appservice.matrix_event_handler
+    async def handle_appservice_event(evt: Event) -> None:
+        if isinstance(evt.content, BaseMessageEventContentFuncs):
+            evt.content.trim_reply_fallback()
+        fake_sync_stream = SyncStream.JOINED_ROOM
+        if evt.type.is_ephemeral:
+            fake_sync_stream |= SyncStream.EPHEMERAL
+        else:
+            fake_sync_stream |= SyncStream.TIMELINE
+        setattr(evt, "source", fake_sync_stream)
+        tasks = client.dispatch_manual_event(evt.type, evt, include_global_handlers=True)
+        await asyncio.gather(*tasks)
 
 
 async def main():
@@ -217,6 +254,8 @@ async def main():
         state_store=state_store,
         device_id=device_id,
     )
+    if appservice:
+        client.api.as_user_id = user_id
     client.ignore_first_sync = config["user.ignore_first_sync"]
     client.ignore_initial_sync = config["user.ignore_initial_sync"]
     if crypto_store:
@@ -225,6 +264,11 @@ async def main():
         await crypto_store.open()
 
         client.crypto = OlmMachine(client, crypto_store, state_store)
+        if appservice:
+            appservice.otk_handler = client.crypto.handle_as_otk_counts
+            appservice.device_list_handler = client.crypto.handle_as_device_lists
+            appservice.to_device_handler = client.crypto.handle_as_to_device_event
+            client.api.as_device_id = device_id
         crypto_device_id = await crypto_store.get_device_id()
         if crypto_device_id and crypto_device_id != device_id:
             log.fatal(
@@ -272,6 +316,8 @@ async def main():
             )
             await nb.put_filter_id(filter_id)
         _ = client.start(nb.filter_id)
+    elif appservice_listener and crypto_store and not client.crypto.account.shared:
+        await client.crypto.share_keys()
 
     if config["user.autojoin"]:
         log.debug("Autojoin is enabled")
@@ -334,9 +380,14 @@ async def stop(suppress_stop_error: bool = False) -> None:
         except Exception:
             if not suppress_stop_error:
                 log.exception("Error stopping bot")
-    if web_runner:
-        await web_runner.shutdown()
-        await web_runner.cleanup()
+    if web_runner and web_runner.server:
+        try:
+            await web_runner.shutdown()
+            await web_runner.cleanup()
+        except RuntimeError:
+            if not suppress_stop_error:
+                await db.stop()
+                raise
     await db.stop()
 
 
@@ -347,6 +398,10 @@ signal.signal(signal.SIGTERM, signal.default_int_handler)
 try:
     log.info("Starting plugin")
     loop.run_until_complete(main())
+except SystemExit:
+    loop.run_until_complete(stop(suppress_stop_error=True))
+    loop.close()
+    raise
 except (Exception, KeyboardInterrupt) as e:
     if isinstance(e, KeyboardInterrupt):
         log.info("Startup interrupted, stopping")

maubot/standalone/config.py

@@ -33,9 +33,13 @@ class Config(BaseFileConfig):
         copy("user.credentials.access_token")
         copy("user.credentials.device_id")
         copy("user.sync")
+        copy("user.appservice")
+        copy("user.hs_token")
         copy("user.autojoin")
         copy("user.displayname")
         copy("user.avatar_url")
+        copy("user.ignore_initial_sync")
+        copy("user.ignore_first_sync")
         if "server" in base:
             copy("server.hostname")
             copy("server.port")

maubot/standalone/example-config.yaml

@@ -5,9 +5,15 @@ user:
         homeserver: https://example.com
         access_token: foo
         # If you want to enable encryption, set the device ID corresponding to the access token here.
+        # When using an appservice, you should use appservice login manually to generate a device ID and access token.
         device_id: null
     # Enable /sync? This is not needed for purely unencrypted webhook-based bots, but is necessary in most other cases.
     sync: true
+    # Receive appservice transactions? This will add a /_matrix/app/v1/transactions endpoint on
+    # the HTTP server configured below. The base_path will not be applied for the /transactions path.
+    appservice: false
+    # When appservice mode is enabled, the hs_token for the appservice.
+    hs_token: null
     # Automatically accept invites?
     autojoin: false
     # The displayname and avatar URL to set for the bot on startup.
@@ -21,7 +27,8 @@ user:
     # if you want the bot to handle messages that were sent while the bot was down.
     ignore_first_sync: true
 
-# Web server settings. These will only take effect if the plugin requests it using `webapp: true` in the meta file.
+# Web server settings. These will only take effect if the plugin requests it using `webapp: true` in the meta file,
+# or if user -> appservice is set to true.
 server:
     # The IP and port to listen to.
     hostname: 0.0.0.0
@@ -35,7 +42,7 @@ server:
 
 # The database for the plugin. Used for plugin data, the sync token and e2ee data (if enabled).
 # SQLite and Postgres are supported.
-database: sqlite:///bot.db
+database: sqlite:bot.db
 
 # Additional arguments for asyncpg.create_pool() or sqlite3.connect()
 # https://magicstack.github.io/asyncpg/current/api/index.html#asyncpg.pool.create_pool

maubot/standalone/loader.py

@@ -18,12 +18,13 @@ from __future__ import annotations
 import os
 import os.path
 
-from ..loader import BasePluginLoader
+from ..loader import BasePluginLoader, PluginMeta
 
 
 class FileSystemLoader(BasePluginLoader):
-    def __init__(self, path: str) -> None:
+    def __init__(self, path: str, meta: PluginMeta) -> None:
         self.path = path
+        self.meta = meta
 
     @property
     def source(self) -> str:

pyproject.toml

@@ -10,5 +10,4 @@ skip = ["maubot/management/frontend"]
 [tool.black]
 line-length = 99
 target-version = ["py38"]
-required-version = "22.1.0"
 force-exclude = "maubot/management/frontend"

requirements.txt

@@ -1,17 +1,16 @@
-mautrix>=0.15.5,<0.16
+mautrix>=0.20.2,<0.21
 aiohttp>=3,<4
 yarl>=1,<2
 SQLAlchemy>=1,<1.4
-asyncpg>=0.20,<0.26
-aiosqlite>=0.16,<0.18
-alembic>=1,<2
+asyncpg>=0.20,<0.29
+aiosqlite>=0.16,<0.19
 commonmark>=0.9,<1
 ruamel.yaml>=0.15.35,<0.18
 attrs>=18.1.0
-bcrypt>=3,<4
+bcrypt>=3,<5
 packaging>=10
 
-click>=7,<8
+click>=7,<9
 colorama>=0.4,<0.5
 questionary>=1,<2
 jinja2>=2,<4

setup.py

@@ -41,7 +41,7 @@ setuptools.setup(
 
     install_requires=install_requires,
     extras_require=extras_require,
-    python_requires="~=3.8",
+    python_requires="~=3.9",
 
     classifiers=[
         "Development Status :: 4 - Beta",
@@ -50,9 +50,9 @@ setuptools.setup(
         "Framework :: AsyncIO",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
     ],
     entry_points="""
         [console_scripts]
@@ -68,6 +68,7 @@ setuptools.setup(
             "management/frontend/build/static/css/*",
             "management/frontend/build/static/js/*",
             "management/frontend/build/static/media/*",
+            "py.typed",
         ],
         "maubot.cli": ["res/*"],
         "maubot.standalone": ["example-config.yaml"],