vb: update nodejs in local build

[skip ci]

.github/workflows/python-lint.yml

@@ -6,16 +6,17 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v3
+    - uses: actions/setup-python@v5
       with:
-        python-version: "3.10"
+        python-version: "3.13"
     - uses: isort/isort-action@master
       with:
         sortPaths: "./maubot"
     - uses: psf/black@stable
       with:
         src: "./maubot"
+        version: "24.10.0"
     - name: pre-commit
       run: |
         pip install pre-commit

.gitignore

@@ -7,11 +7,13 @@ pip-selfcheck.json
 *.pyc
 __pycache__
 
-*.db
+*.db*
+*.log
 /*.yaml
 !example-config.yaml
 !.pre-commit-config.yaml
 
+/start
 logs/
 plugins/
 trash/

.gitlab-ci-plugin.yml

@@ -0,0 +1,29 @@
+image: dock.mau.dev/maubot/maubot
+
+stages:
+- build
+
+variables:
+  PYTHONPATH: /opt/maubot
+
+build:
+  stage: build
+  except:
+  - tags
+  script:
+  - python3 -m maubot.cli build -o xyz.maubot.$CI_PROJECT_NAME-$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA.mbp
+  artifacts:
+    paths:
+    - "*.mbp"
+    expire_in: 365 days
+
+build tags:
+  stage: build
+  only:
+  - tags
+  script:
+  - python3 -m maubot.cli build -o xyz.maubot.$CI_PROJECT_NAME-$CI_COMMIT_TAG.mbp
+  artifacts:
+    paths:
+    - "*.mbp"
+    expire_in: never

.gitlab-ci.yml

@@ -10,7 +10,7 @@ default:
   - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
 
 build frontend:
-  image: node:16-alpine
+  image: node:22-alpine
   stage: build frontend
   before_script: []
   variables:

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
         exclude_types: [markdown]
@@ -8,13 +8,13 @@ repos:
       - id: check-yaml
       - id: check-added-large-files
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 24.10.0
     hooks:
       - id: black
         language_version: python3
         files: ^maubot/.*\.pyi?$
   - repo: https://github.com/PyCQA/isort
-    rev: 5.10.1
+    rev: 5.13.2
     hooks:
       - id: isort
-        files: ^maubot/.*\.pyi$
+        files: ^maubot/.*\.pyi?$

CHANGELOG.md

@@ -1,3 +1,84 @@
+# v0.5.2 (2025-05-05)
+
+* Improved tombstone handling to ensure that the tombstone sender has
+  permissions to invite users to the target room.
+* Fixed autojoin and online flags not being applied if set during client
+  creation (thanks to [@bnsh] in [#258]).
+* Fixed plugin web apps not being cleared properly when unloading plugins.
+
+[@bnsh]: https://github.com/bnsh
+[#258]: https://github.com/maubot/maubot/pull/258
+
+# v0.5.1 (2025-01-03)
+
+* Updated Docker image to Alpine 3.21.
+* Updated media upload/download endpoints in management frontend
+  (thanks to [@domrim] in [#253]).
+* Fixed plugin web app base path not including a trailing slash
+  (thanks to [@jkhsjdhjs] in [#240]).
+* Changed markdown parsing to cut off plaintext body if necessary to allow
+  longer formatted messages.
+* Updated dependencies to fix Python 3.13 compatibility.
+
+[@domrim]: https://github.com/domrim
+[@jkhsjdhjs]: https://github.com/jkhsjdhjs
+[#253]: https://github.com/maubot/maubot/pull/253
+[#240]: https://github.com/maubot/maubot/pull/240
+
+# v0.5.0 (2024-08-24)
+
+* Dropped Python 3.9 support.
+* Updated Docker image to Alpine 3.20.
+* Updated mautrix-python to 0.20.6 to support authenticated media.
+* Removed hard dependency on SQLAlchemy.
+* Fixed `main_class` to default to being loaded from the last module instead of
+  the first if a module name is not explicitly specified.
+  * This was already the [documented behavior](https://docs.mau.fi/maubot/dev/reference/plugin-metadata.html),
+    and loading from the first module doesn't make sense due to import order.
+* Added simple scheduler utility for running background tasks periodically or
+  after a certain delay.
+* Added testing framework for plugins (thanks to [@abompard] in [#225]).
+* Changed `mbc build` to ignore directories declared in `modules` that are
+  missing an `__init__.py` file.
+  * Importing the modules at runtime would fail and break the plugin.
+    To include non-code resources outside modules in the mbp archive,
+    use `extra_files` instead.
+
+[#225]: https://github.com/maubot/maubot/issues/225
+[@abompard]: https://github.com/abompard
+
+# v0.4.2 (2023-09-20)
+
+* Updated Pillow to 10.0.1.
+* Updated Docker image to Alpine 3.18.
+* Added logging for errors for /whoami errors when adding new bot accounts.
+* Added support for using appservice tokens (including appservice encryption)
+  in standalone mode.
+
+# v0.4.1 (2023-03-15)
+
+* Added `in_thread` parameter to `evt.reply()` and `evt.respond()`.
+  * By default, responses will go to the thread if the command is in a thread.
+  * By setting the flag to `True` or `False`, the plugin can force the response
+    to either be or not be in a thread.
+* Fixed static files like the frontend app manifest not being served correctly.
+* Fixed `self.loader.meta` not being available to plugins in standalone mode.
+* Updated to mautrix-python v0.19.6.
+
+# v0.4.0 (2023-01-29)
+
+* Dropped support for using a custom maubot API base path.
+  * The public URL can still have a path prefix, e.g. when using a reverse
+    proxy. Both the web interface and `mbc` CLI tool should work fine with
+    custom prefixes.
+* Added `evt.redact()` as a shortcut for `self.client.redact(evt.room_id, evt.event_id)`.
+* Fixed `mbc logs` command not working on Python 3.8+.
+* Fixed saving plugin configs (broke in v0.3.0).
+* Fixed SSO login using the wrong API path (probably broke in v0.3.0).
+* Stopped using `cd` in the docker image's `mbc` wrapper to enable using
+  path-dependent commands like `mbc build` by mounting a directory.
+* Updated Docker image to Alpine 3.17.
+
 # v0.3.1 (2022-03-29)
 
 * Added encryption dependencies to standalone dockerfile.

Dockerfile

@@ -1,9 +1,9 @@
-FROM node:16 AS frontend-builder
+FROM node:22 AS frontend-builder
 
 COPY ./maubot/management/frontend /frontend
 RUN cd /frontend && yarn --prod && yarn build
 
-FROM alpine:3.15
+FROM alpine:3.21
 
 RUN apk add --no-cache \
         python3 py3-pip py3-setuptools py3-wheel \
@@ -11,7 +11,6 @@ RUN apk add --no-cache \
         su-exec \
         yq \
         py3-aiohttp \
-        py3-sqlalchemy \
         py3-attrs \
         py3-bcrypt \
         py3-cffi \
@@ -21,11 +20,10 @@ RUN apk add --no-cache \
         py3-packaging \
         py3-markdown \
         py3-alembic \
-#        py3-cssselect \
+        py3-cssselect \
         py3-commonmark \
         py3-pygments \
         py3-tz \
-#        py3-tzlocal \
         py3-regex \
         py3-wcwidth \
         # encryption
@@ -39,17 +37,16 @@ RUN apk add --no-cache \
         py3-magic \
         py3-feedparser \
         py3-dateutil \
-        py3-lxml
+        py3-lxml \
-#        py3-gitlab
+        py3-semver
-#        py3-semver@edge
 # TODO remove pillow, magic, feedparser, lxml, gitlab and semver when maubot supports installing dependencies
 
 COPY requirements.txt /opt/maubot/requirements.txt
 COPY optional-requirements.txt /opt/maubot/optional-requirements.txt
 WORKDIR /opt/maubot
 RUN apk add --virtual .build-deps python3-dev build-base git \
-    && pip3 install -r requirements.txt -r optional-requirements.txt \
+    && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \
-        dateparser langdetect python-gitlab pyquery cchardet semver tzlocal cssselect \
+        dateparser langdetect python-gitlab pyquery tzlocal \
     && apk del .build-deps
 # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies
 

Dockerfile.ci

@@ -1,4 +1,4 @@
-FROM alpine:3.15
+FROM alpine:3.21
 
 RUN apk add --no-cache \
         python3 py3-pip py3-setuptools py3-wheel \
@@ -6,7 +6,6 @@ RUN apk add --no-cache \
         su-exec \
         yq \
         py3-aiohttp \
-        py3-sqlalchemy \
         py3-attrs \
         py3-bcrypt \
         py3-cffi \
@@ -42,8 +41,8 @@ COPY requirements.txt /opt/maubot/requirements.txt
 COPY optional-requirements.txt /opt/maubot/optional-requirements.txt
 WORKDIR /opt/maubot
 RUN apk add --virtual .build-deps python3-dev build-base git \
-    && pip3 install -r requirements.txt -r optional-requirements.txt \
+    && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \
-        dateparser langdetect python-gitlab pyquery cchardet semver tzlocal cssselect \
+        dateparser langdetect python-gitlab pyquery semver tzlocal cssselect \
     && apk del .build-deps
 # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies
 

Dockerfile.local

@@ -0,0 +1,29 @@
+FROM r.batts.cloud/nodejs:22 AS frontend-builder
+
+COPY ./maubot/management/frontend /frontend
+RUN cd /frontend && yarn --prod && yarn build
+
+FROM r.batts.cloud/debian:bookworm
+
+RUN apt update && \
+    apt install -y --no-install-recommends python3 python3-dev python3-venv python3-semver git gosu yq brotli && \
+    apt clean -y && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /opt/maubot/requirements.txt
+COPY optional-requirements.txt /opt/maubot/optional-requirements.txt
+WORKDIR /opt/maubot
+RUN python3 -m venv /venv \
+    && bash -c 'source /venv/bin/activate \
+    && pip3 install -r requirements.txt -r optional-requirements.txt \
+        dateparser langdetect python-gitlab pyquery tzlocal pyfiglet emoji feedparser brotli'
+# TODO also remove pyfiglet, emoji, dateparser, langdetect and pyquery when maubot supports installing dependencies
+
+COPY . /opt/maubot
+RUN cp maubot/example-config.yaml .
+COPY ./docker/mbc.sh /usr/local/bin/mbc
+COPY --from=frontend-builder /frontend/build /opt/maubot/frontend
+ENV UID=1337 GID=1337 XDG_CONFIG_HOME=/data
+VOLUME /data
+
+CMD ["/opt/maubot/docker/run.sh"]

README.md

@@ -22,56 +22,8 @@ All setup and usage instructions are located on
 Matrix room: [#maubot:maunium.net](https://matrix.to/#/#maubot:maunium.net)
 
 ## Plugins
-Open a pull request or join the Matrix room linked above to get your plugin listed here.
+A list of plugins can be found at [plugins.mau.bot](https://plugins.mau.bot/).
+
+To add your plugin to the list, send a pull request to <https://github.com/maubot/plugins.maubot.xyz>.
 
 The plugin wishlist lives at <https://github.com/maubot/plugin-wishlist/issues>.
-
-### Official plugins
-* [sed](https://github.com/maubot/sed) - A bot to do sed-like replacements.
-* [factorial](https://github.com/maubot/factorial) - A bot to calculate unexpected factorials.
-* [media](https://github.com/maubot/media) - A bot that replies with the MXC URI of images you send it.
-* [dice](https://github.com/maubot/dice) - A combined dice rolling and calculator bot.
-* [karma](https://github.com/maubot/karma) - A user karma tracker bot.
-* [xkcd](https://github.com/maubot/xkcd) - A bot to view xkcd comics.
-* [echo](https://github.com/maubot/echo) - A bot that echoes pings and other stuff.
-* [rss](https://github.com/maubot/rss) - A bot that posts RSS feed updates to Matrix.
-* [reminder](https://github.com/maubot/reminder) - A bot to remind you about things.
-* [translate](https://github.com/maubot/translate) - A bot to translate words.
-* [reactbot](https://github.com/maubot/reactbot) - A bot that responds to messages that match predefined rules.
-* [exec](https://github.com/maubot/exec) - A bot that executes code.
-* [commitstrip](https://github.com/maubot/commitstrip) - A bot to view CommitStrips.
-* [supportportal](https://github.com/maubot/supportportal) - A bot to manage customer support on Matrix.
-* †[gitlab](https://github.com/maubot/gitlab) - A GitLab client and webhook receiver.
-* [github](https://github.com/maubot/github) - A GitHub client and webhook receiver.
-* [tex](https://github.com/maubot/tex) - A bot that renders LaTeX.
-* [altalias](https://github.com/maubot/altalias) - A bot that lets users publish alternate aliases in rooms.
-* [satwcomic](https://github.com/maubot/satwcomic) - A bot to view SatWComics.
-* [songwhip](https://github.com/maubot/songwhip) - A bot to post Songwhip links.
-* [manhole](https://github.com/maubot/manhole) - A plugin that lets you access a Python shell inside maubot.
-
-### 3rd party plugins
-* [subreddit linkifier](https://github.com/TomCasavant/RedditMaubot) - A bot that condescendingly corrects a user when they enter an r/subreddit without providing a link to that subreddit
-* [giphy](https://github.com/TomCasavant/GiphyMaubot) - A bot that generates a gif (from giphy) given search terms
-* [trump](https://github.com/jeffcasavant/MaubotTrumpTweet) - A bot that generates a Trump tweet with the given content
-* [poll](https://github.com/TomCasavant/PollMaubot) - A bot that will create a simple poll for users in a room
-* [urban](https://github.com/dvdgsng/UrbanMaubot) - A bot that fetches definitions from [Urban Dictionary](https://www.urbandictionary.com/).
-* [twilio](https://github.com/jeffcasavant/MaubotTwilio) - Maubot-based SMS bridge
-* [tmdb](https://codeberg.org/lomion/tmdb-bot) - A bot that posts information about movies fetched from TheMovieDB.org.
-* [invite](https://github.com/williamkray/maubot-invite) - A bot to generate invitation tokens from [matrix-registration](https://github.com/ZerataX/matrix-registration)
-* [wolframalpha](https://github.com/ggogel/WolframAlphaMaubot) - A bot that allows requesting information from [WolframAlpha](https://www.wolframalpha.com/).
-* †[pingcheck](https://edugit.org/nik/maubot-pingcheck) - A bot to ping the echo bot and send rtt to Icinga passive check
-* [ticker](https://github.com/williamkray/maubot-ticker) - A bot to return financial data about a stock or cryptocurrency.
-* [weather](https://github.com/kellya/maubot-weather) - A bot to get the weather from wttr.in and return a single line of text for the location specified
-* †[youtube previewer](https://github.com/ggogel/YoutubePreviewMaubot) - A bot that responds to a YouTube link with the video title and thumbnail.
-* †[reddit previewer](https://github.com/ggogel/RedditPreviewMaubot) - A bot that responds to a link of a reddit post with the sub name and title. If available, uploads the image or video.
-* [pocket](https://github.com/jaywink/maubot-pocket) - A bot integrating with Pocket to fetch articles and archive them.
-* [alternatingcaps](https://github.com/rom4nik/maubot-alternatingcaps) - A bot repeating last message using aLtErNaTiNg cApS.
-* [metric](https://github.com/edwardsdean/maubot_metric_bot) - A bot that will reply to a message that contains imperial units and replace them with metric units.
-* [urlpreview](https://github.com/coffeebank/coffee-maubot/tree/master/urlpreview) - A bot that responds to links with a link preview embed, using Matrix API to fetch meta tags
-
-† Uses a synchronous library which can block the whole maubot process (e.g. requests instead of aiohttp)
-
-### Deprecated/unmaintained plugins
-* [jesaribot](https://github.com/maubot/jesaribot) - A simple bot that replies with an image when you say "jesari".
-  * Superseded by reactbot
-* [gitea](https://github.com/saces/maugitea) - A Gitea client and webhook receiver.

dev-requirements.txt

@@ -1,3 +1,3 @@
 pre-commit>=2.10.1,<3
 isort>=5.10.1,<6
-black>=22.3.0,<22
+black>=24,<25

docker/mbc.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
-cd /opt/maubot
+export PYTHONPATH=/opt/maubot
 python3 -m maubot.cli "$@"

docker/run.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 function fixperms {
 	chown -R $UID:$GID /var/log /data
@@ -13,7 +13,7 @@ function fixdefault {
 
 function fixconfig {
 	# Change relative default paths to absolute paths in /data
-	fixdefault '.database' 'sqlite:///maubot.db' 'sqlite:////data/maubot.db'
+	fixdefault '.database' 'sqlite:maubot.db' 'sqlite:/data/maubot.db'
 	fixdefault '.plugin_directories.upload' './plugins' '/data/plugins'
 	fixdefault '.plugin_directories.load[0]' './plugins' '/data/plugins'
 	fixdefault '.plugin_directories.trash' './trash' '/data/trash'
@@ -30,7 +30,6 @@ mkdir -p /var/log/maubot /data/plugins /data/trash /data/dbs
 
 if [ ! -f /data/config.yaml ]; then
 	cp example-config.yaml /data/config.yaml
-	# Apply some docker-specific adjustments to the config
 	echo "Config file not found. Example config copied to /data/config.yaml"
 	echo "Please modify the config file to your liking and restart the container."
 	fixperms
@@ -38,11 +37,14 @@ if [ ! -f /data/config.yaml ]; then
 	exit
 fi
 
-alembic -x config=/data/config.yaml upgrade head
 fixperms
 fixconfig
 if ls /data/plugins/*.db > /dev/null 2>&1; then
 	mv -n /data/plugins/*.db /data/dbs/
 fi
 
+if [ -f "/venv/bin/activate" ] ; then
+	exec gosu $UID:$GID bash -c 'source /venv/bin/activate && python3 -m maubot -c /data/config.yaml'
+fi
 exec su-exec $UID:$GID python3 -m maubot -c /data/config.yaml
+

maubot/__meta__.py

@@ -1 +1 @@
-__version__ = "0.3.1"
+__version__ = "0.5.2"

maubot/cli/commands/build.py

@@ -93,10 +93,16 @@ def write_plugin(meta: PluginMeta, output: str | IO) -> None:
             if os.path.isfile(f"{module}.py"):
                 zip.write(f"{module}.py")
             elif module is not None and os.path.isdir(module):
-                zipdir(zip, module)
+                if os.path.isfile(f"{module}/__init__.py"):
+                    zipdir(zip, module)
+                else:
+                    print(
+                        Fore.YELLOW
+                        + f"Module {module} is missing __init__.py, skipping"
+                        + Fore.RESET
+                    )
             else:
                 print(Fore.YELLOW + f"Module {module} not found, skipping" + Fore.RESET)
-
         for pattern in meta.extra_files:
             for file in glob.iglob(pattern):
                 zip.write(file)

maubot/cli/commands/logs.py

@@ -38,13 +38,7 @@ def logs(server: str, tail: int) -> None:
     global history_count
     history_count = tail
     loop = asyncio.get_event_loop()
-    future = asyncio.create_task(view_logs(server, token), loop=loop)
+    loop.run_until_complete(view_logs(server, token))
-    try:
-        loop.run_until_complete(future)
-    except KeyboardInterrupt:
-        future.cancel()
-        loop.run_until_complete(future)
-        loop.close()
 
 
 def parsedate(entry: Obj) -> None:

maubot/cli/config.py

@@ -36,6 +36,7 @@ def get_default_server() -> tuple[str | None, str | None]:
         server = None
     if server is None:
         print(f"{Fore.RED}Default server not configured.{Fore.RESET}")
+        print(f"Perhaps you forgot to {Fore.CYAN}mbc login{Fore.RESET}?")
         return None, None
     return server, _get_token(server)
 

maubot/cli/util/spdx.py

@@ -36,10 +36,10 @@ def load() -> None:
 def get(id: str) -> dict[str, str]:
     if not spdx_list:
         load()
-    return spdx_list[id.lower()]
+    return spdx_list[id]
 
 
 def valid(id: str) -> bool:
     if not spdx_list:
         load()
-    return id.lower() in spdx_list
+    return id in spdx_list

maubot/client.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, AsyncGenerator, AsyncIterable, Awaitable, Callable, cast
+from typing import TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable, cast
 from collections import defaultdict
 import asyncio
 import logging
@@ -41,6 +41,7 @@ from mautrix.types import (
     SyncToken,
     UserID,
 )
+from mautrix.util import background_task
 from mautrix.util.async_getter_lock import async_getter_lock
 from mautrix.util.logging import TraceLogger
 
@@ -254,7 +255,7 @@ class Client(DBClient):
                 self.log.warning(
                     f"Failed to get /account/whoami, retrying in {(try_n + 1) * 10}s: {e}"
                 )
-                _ = asyncio.create_task(self.start(try_n + 1))
+                background_task.create(self.start(try_n + 1))
             return
         if whoami.user_id != self.id:
             self.log.error(f"User ID mismatch: expected {self.id}, but got {whoami.user_id}")
@@ -349,11 +350,38 @@ class Client(DBClient):
         }
 
     async def _handle_tombstone(self, evt: StateEvent) -> None:
+        if evt.state_key != "":
+            return
         if not evt.content.replacement_room:
             self.log.info(f"{evt.room_id} tombstoned with no replacement, ignoring")
             return
+        is_joined = await self.client.state_store.is_joined(
+            evt.content.replacement_room,
+            self.client.mxid,
+        )
+        if is_joined:
+            self.log.debug(
+                f"Ignoring tombstone from {evt.room_id} to {evt.content.replacement_room} "
+                f"sent by {evt.sender}: already joined to replacement room"
+            )
+            return
+        self.log.debug(
+            f"Following tombstone from {evt.room_id} to {evt.content.replacement_room} "
+            f"sent by {evt.sender}"
+        )
         _, server = self.client.parse_user_id(evt.sender)
-        await self.client.join_room(evt.content.replacement_room, servers=[server])
+        room_id = await self.client.join_room(evt.content.replacement_room, servers=[server])
+        power_levels = await self.client.get_state_event(room_id, EventType.ROOM_POWER_LEVELS)
+        if power_levels.get_user_level(evt.sender) < power_levels.invite:
+            self.log.warning(
+                f"{evt.room_id} was tombstoned into {room_id} by {evt.sender},"
+                " but the sender doesn't have invite power levels, leaving..."
+            )
+            await self.client.leave_room(
+                room_id,
+                f"Followed tombstone from {evt.room_id} by {evt.sender},"
+                " but sender doesn't have sufficient power level for invites",
+            )
 
     async def _handle_invite(self, evt: StrippedStateEvent) -> None:
         if evt.state_key == self.id and evt.content.membership == Membership.INVITE:

maubot/config.py

@@ -32,7 +32,11 @@ class Config(BaseFileConfig):
     def do_update(self, helper: ConfigUpdateHelper) -> None:
         base = helper.base
         copy = helper.copy
-        copy("database")
+
+        if "database" in self and self["database"].startswith("sqlite:///"):
+            helper.base["database"] = self["database"].replace("sqlite:///", "sqlite:")
+        else:
+            copy("database")
         copy("database_opts")
         if isinstance(self["crypto_database"], dict):
             if self["crypto_database.type"] == "postgres":
@@ -52,11 +56,9 @@ class Config(BaseFileConfig):
         copy("server.port")
         copy("server.public_url")
         copy("server.listen")
-        copy("server.base_path")
         copy("server.ui_base_path")
         copy("server.plugin_base_path")
         copy("server.override_resource_path")
-        copy("server.appservice_base_path")
         shared_secret = self["server.unshared_secret"]
         if shared_secret is None or shared_secret == "generate":
             base["server.unshared_secret"] = self._new_token()

maubot/db/__init__.py

@@ -1,7 +1,7 @@
 from mautrix.util.async_db import Database
 
 from .client import Client
-from .instance import Instance
+from .instance import DatabaseEngine, Instance
 from .upgrade import upgrade_table
 
 
@@ -10,4 +10,4 @@ def init(db: Database) -> None:
         table.db = db
 
 
-__all__ = ["upgrade_table", "init", "Client", "Instance"]
+__all__ = ["upgrade_table", "init", "Client", "Instance", "DatabaseEngine"]

maubot/db/instance.py

@@ -16,6 +16,7 @@
 from __future__ import annotations
 
 from typing import TYPE_CHECKING, ClassVar
+from enum import Enum
 
 from asyncpg import Record
 from attr import dataclass
@@ -26,6 +27,11 @@ from mautrix.util.async_db import Database
 fake_db = Database.create("") if TYPE_CHECKING else None
 
 
+class DatabaseEngine(Enum):
+    SQLITE = "sqlite"
+    POSTGRES = "postgres"
+
+
 @dataclass
 class Instance:
     db: ClassVar[Database] = fake_db
@@ -35,21 +41,31 @@ class Instance:
     enabled: bool
     primary_user: UserID
     config_str: str
+    database_engine: DatabaseEngine | None
+
+    @property
+    def database_engine_str(self) -> str | None:
+        return self.database_engine.value if self.database_engine else None
 
     @classmethod
     def _from_row(cls, row: Record | None) -> Instance | None:
         if row is None:
             return None
-        return cls(**row)
+        data = {**row}
+        db_engine = data.pop("database_engine", None)
+        return cls(**data, database_engine=DatabaseEngine(db_engine) if db_engine else None)
+
+    _columns = "id, type, enabled, primary_user, config, database_engine"
 
     @classmethod
     async def all(cls) -> list[Instance]:
-        rows = await cls.db.fetch("SELECT id, type, enabled, primary_user, config FROM instance")
+        q = f"SELECT {cls._columns} FROM instance"
+        rows = await cls.db.fetch(q)
         return [cls._from_row(row) for row in rows]
 
     @classmethod
     async def get(cls, id: str) -> Instance | None:
-        q = "SELECT id, type, enabled, primary_user, config FROM instance WHERE id=$1"
+        q = f"SELECT {cls._columns} FROM instance WHERE id=$1"
         return cls._from_row(await cls.db.fetchrow(q, id))
 
     async def update_id(self, new_id: str) -> None:
@@ -58,17 +74,27 @@ class Instance:
 
     @property
     def _values(self):
-        return self.id, self.type, self.enabled, self.primary_user, self.config_str
+        return (
+            self.id,
+            self.type,
+            self.enabled,
+            self.primary_user,
+            self.config_str,
+            self.database_engine_str,
+        )
 
     async def insert(self) -> None:
         q = (
-            "INSERT INTO instance (id, type, enabled, primary_user, config) "
+            "INSERT INTO instance (id, type, enabled, primary_user, config, database_engine) "
-            "VALUES ($1, $2, $3, $4, $5)"
+            "VALUES ($1, $2, $3, $4, $5, $6)"
         )
         await self.db.execute(q, *self._values)
 
     async def update(self) -> None:
-        q = "UPDATE instance SET type=$2, enabled=$3, primary_user=$4, config=$5 WHERE id=$1"
+        q = """
+        UPDATE instance SET type=$2, enabled=$3, primary_user=$4, config=$5, database_engine=$6
+        WHERE id=$1
+        """
         await self.db.execute(q, *self._values)
 
     async def delete(self) -> None:

maubot/db/upgrade/__init__.py

@@ -2,4 +2,4 @@ from mautrix.util.async_db import UpgradeTable
 
 upgrade_table = UpgradeTable()
 
-from . import v01_initial_revision
+from . import v01_initial_revision, v02_instance_database_engine

maubot/db/upgrade/v02_instance_database_engine.py

@@ -0,0 +1,25 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2022 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from mautrix.util.async_db import Connection
+
+from . import upgrade_table
+
+
+@upgrade_table.register(description="Store instance database engine")
+async def upgrade_v2(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE instance ADD COLUMN database_engine TEXT")

maubot/example-config.yaml

@@ -1,9 +1,8 @@
 # The full URI to the database. SQLite and Postgres are fully supported.
-# Other DBMSes supported by SQLAlchemy may or may not work.
 # Format examples:
-#   SQLite:   sqlite:///filename.db
+#   SQLite:   sqlite:filename.db
 #   Postgres: postgresql://username:password@hostname/dbname
-database: sqlite:///maubot.db
+database: sqlite:maubot.db
 
 # Separate database URL for the crypto database. "default" means use the same database as above.
 crypto_database: default
@@ -55,8 +54,6 @@ server:
     port: 29316
     # Public base URL where the server is visible.
     public_url: https://example.com
-    # The base management API path.
-    base_path: /_matrix/maubot/v1
     # The base path for the UI.
     ui_base_path: /_matrix/maubot
     # The base path for plugin endpoints. The instance ID will be appended directly.
@@ -64,8 +61,6 @@ server:
     # Override path from where to load UI resources.
     # Set to false to using pkg_resources to find the path.
     override_resource_path: false
-    # The base appservice API path. Use / for legacy appservice API and /_matrix/app/v1 for v1.
-    appservice_base_path: /_matrix/app/v1
     # The shared secret to sign API access tokens.
     # Set to "generate" to generate and save a new token at startup.
     unshared_secret: generate
@@ -83,8 +78,9 @@ homeservers:
         # When this is empty, `mbc auth --register` won't work, but `mbc auth` (login) will.
         secret: null
 
-# List of administrator users. Plaintext passwords will be bcrypted on startup. Set empty password
+# List of administrator users. Each key is a username and the value is the password.
-# to prevent normal login. Root is a special user that can't have a password and will always exist.
+# Plaintext passwords will be bcrypted on startup. Set empty password to prevent normal login.
+# Root is a special user that can't have a password and will always exist.
 admins:
     root: ""
 

maubot/handlers/command.py

@@ -72,7 +72,7 @@ class CommandHandler:
         self.__mb_must_consume_args__: bool = True
         self.__mb_arg_fallthrough__: bool = True
         self.__mb_event_handler__: bool = True
-        self.__mb_event_type__: EventType = EventType.ROOM_MESSAGE
+        self.__mb_event_types__: set[EventType] = {EventType.ROOM_MESSAGE}
         self.__mb_msgtypes__: Iterable[MessageType] = (MessageType.TEXT,)
         self.__bound_copies__: Dict[Any, CommandHandler] = {}
         self.__bound_instance__: Any = None
@@ -92,9 +92,10 @@ class CommandHandler:
                 "get_name",
                 "is_command_match",
                 "require_subcommand",
+                "must_consume_args",
                 "arg_fallthrough",
                 "event_handler",
-                "event_type",
+                "event_types",
                 "msgtypes",
             ]
             for key in keys:
@@ -314,7 +315,7 @@ def new(
         func.__mb_require_subcommand__ = require_subcommand
         func.__mb_arg_fallthrough__ = arg_fallthrough
         func.__mb_must_consume_args__ = must_consume_args
-        func.__mb_event_type__ = event_type
+        func.__mb_event_types__ = {event_type}
         if msgtypes:
             func.__mb_msgtypes__ = msgtypes
         return func

maubot/handlers/event.py

@@ -27,9 +27,12 @@ def on(var: EventType | InternalEventType | EventHandler) -> EventHandlerDecorat
     def decorator(func: EventHandler) -> EventHandler:
         func.__mb_event_handler__ = True
         if isinstance(var, (EventType, InternalEventType)):
-            func.__mb_event_type__ = var
+            if hasattr(func, "__mb_event_types__"):
+                func.__mb_event_types__.add(var)
+            else:
+                func.__mb_event_types__ = {var}
         else:
-            func.__mb_event_type__ = EventType.ALL
+            func.__mb_event_types__ = {EventType.ALL}
 
         return func
 

maubot/instance.py

@@ -25,16 +25,17 @@ import os.path
 
 from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap
-import sqlalchemy as sql
 
 from mautrix.types import UserID
+from mautrix.util import background_task
 from mautrix.util.async_db import Database, Scheme, UpgradeTable
 from mautrix.util.async_getter_lock import async_getter_lock
 from mautrix.util.config import BaseProxyConfig, RecursiveDict
 from mautrix.util.logging import TraceLogger
 
 from .client import Client
-from .db import Instance as DBInstance
+from .db import DatabaseEngine, Instance as DBInstance
+from .lib.optionalalchemy import Engine, MetaData, create_engine
 from .lib.plugin_db import ProxyPostgresDatabase
 from .loader import DatabaseType, PluginLoader, ZippedPluginLoader
 from .plugin_base import Plugin
@@ -71,10 +72,21 @@ class PluginInstance(DBInstance):
     started: bool
 
     def __init__(
-        self, id: str, type: str, enabled: bool, primary_user: UserID, config: str = ""
+        self,
+        id: str,
+        type: str,
+        enabled: bool,
+        primary_user: UserID,
+        config: str = "",
+        database_engine: DatabaseEngine | None = None,
     ) -> None:
         super().__init__(
-            id=id, type=type, enabled=bool(enabled), primary_user=primary_user, config_str=config
+            id=id,
+            type=type,
+            enabled=bool(enabled),
+            primary_user=primary_user,
+            config_str=config,
+            database_engine=database_engine,
         )
 
     def __hash__(self) -> int:
@@ -111,10 +123,12 @@ class PluginInstance(DBInstance):
             "database": (
                 self.inst_db is not None and self.maubot.config["api_features.instance_database"]
             ),
+            "database_interface": self.loader.meta.database_type_str if self.loader else "unknown",
+            "database_engine": self.database_engine_str,
         }
 
     def _introspect_sqlalchemy(self) -> dict:
-        metadata = sql.MetaData()
+        metadata = MetaData()
         metadata.reflect(self.inst_db)
         return {
             table.name: {
@@ -200,7 +214,7 @@ class PluginInstance(DBInstance):
 
     async def get_db_tables(self) -> dict:
         if self.inst_db_tables is None:
-            if isinstance(self.inst_db, sql.engine.Engine):
+            if isinstance(self.inst_db, Engine):
                 self.inst_db_tables = self._introspect_sqlalchemy()
             elif self.inst_db.scheme == Scheme.SQLITE:
                 self.inst_db_tables = await self._introspect_sqlite()
@@ -263,18 +277,37 @@ class PluginInstance(DBInstance):
     def save_config(self, data: RecursiveDict[CommentedMap]) -> None:
         buf = io.StringIO()
         yaml.dump(data, buf)
-        self.config_str = buf.getvalue()
+        val = buf.getvalue()
+        if val != self.config_str:
+            self.config_str = val
+            self.log.debug("Creating background task to save updated config")
+            background_task.create(self.update())
 
     async def start_database(
         self, upgrade_table: UpgradeTable | None = None, actually_start: bool = True
     ) -> None:
         if self.loader.meta.database_type == DatabaseType.SQLALCHEMY:
-            self.inst_db = sql.create_engine(f"sqlite:///{self._sqlite_db_path}")
+            if self.database_engine is None:
+                await self.update_db_engine(DatabaseEngine.SQLITE)
+            elif self.database_engine == DatabaseEngine.POSTGRES:
+                raise RuntimeError(
+                    "Instance database engine is marked as Postgres, but plugin uses legacy "
+                    "database interface, which doesn't support postgres."
+                )
+            self.inst_db = create_engine(f"sqlite:///{self._sqlite_db_path}")
         elif self.loader.meta.database_type == DatabaseType.ASYNCPG:
+            if self.database_engine is None:
+                if os.path.exists(self._sqlite_db_path) or not self.maubot.plugin_postgres_db:
+                    await self.update_db_engine(DatabaseEngine.SQLITE)
+                else:
+                    await self.update_db_engine(DatabaseEngine.POSTGRES)
             instance_db_log = db_log.getChild(self.id)
-            # TODO should there be a way to choose between SQLite and Postgres
+            if self.database_engine == DatabaseEngine.POSTGRES:
-            #      for individual instances? Maybe checking the existence of the SQLite file.
+                if not self.maubot.plugin_postgres_db:
-            if self.maubot.plugin_postgres_db:
+                    raise RuntimeError(
+                        "Instance database engine is marked as Postgres, but this maubot isn't "
+                        "configured to support Postgres for plugin databases"
+                    )
                 self.inst_db = ProxyPostgresDatabase(
                     pool=self.maubot.plugin_postgres_db,
                     instance_id=self.id,
@@ -284,7 +317,7 @@ class PluginInstance(DBInstance):
                 )
             else:
                 self.inst_db = Database.create(
-                    f"sqlite:///{self._sqlite_db_path}",
+                    f"sqlite:{self._sqlite_db_path}",
                     upgrade_table=upgrade_table,
                     log=instance_db_log,
                 )
@@ -296,7 +329,7 @@ class PluginInstance(DBInstance):
     async def stop_database(self) -> None:
         if isinstance(self.inst_db, Database):
             await self.inst_db.stop()
-        elif isinstance(self.inst_db, sql.engine.Engine):
+        elif isinstance(self.inst_db, Engine):
             self.inst_db.dispose()
         else:
             raise RuntimeError(f"Unknown database type {type(self.inst_db).__name__}")
@@ -334,7 +367,12 @@ class PluginInstance(DBInstance):
             self.log.debug("Disabling webapp after plugin meta reload")
             self.disable_webapp()
         if self.loader.meta.database:
-            await self.start_database(cls.get_db_upgrade_table())
+            try:
+                await self.start_database(cls.get_db_upgrade_table())
+            except Exception:
+                self.log.exception("Failed to start instance database")
+                await self.update_enabled(False)
+                return
         config_class = cls.get_config_class()
         if config_class:
             try:
@@ -455,6 +493,11 @@ class PluginInstance(DBInstance):
             self.enabled = enabled
             await self.update()
 
+    async def update_db_engine(self, db_engine: DatabaseEngine | None) -> None:
+        if db_engine is not None and db_engine != self.database_engine:
+            self.database_engine = db_engine
+            await self.update()
+
     @classmethod
     @async_getter_lock
     async def get(

maubot/lib/optionalalchemy.py

@@ -0,0 +1,19 @@
+try:
+    from sqlalchemy import MetaData, asc, create_engine, desc
+    from sqlalchemy.engine import Engine
+    from sqlalchemy.exc import IntegrityError, OperationalError
+except ImportError:
+
+    class FakeError(Exception):
+        pass
+
+    class FakeType:
+        def __init__(self, *args, **kwargs):
+            raise Exception("SQLAlchemy is not installed")
+
+    def create_engine(*args, **kwargs):
+        raise Exception("SQLAlchemy is not installed")
+
+    MetaData = Engine = FakeType
+    IntegrityError = OperationalError = FakeError
+    asc = desc = lambda a: a

maubot/lib/zipimport.py

@@ -323,6 +323,7 @@ _zip_searchorder = (
     (".py", False, False),
 )
 
+
 # Given a module name, return the potential file path in the
 # archive (without extension).
 def _get_module_path(self, fullname):
@@ -351,6 +352,7 @@ def _get_module_info(self, fullname):
 
 # implementation
 
+
 # _read_directory(archive) -> files dict (new reference)
 #
 # Given a path to a Zip archive, build a dict, mapping file names
@@ -524,6 +526,7 @@ cp437_table = (
 
 _importing_zlib = False
 
+
 # Return the zlib.decompress function object, or NULL if zlib couldn't
 # be imported. The function is cached when found, so subsequent calls
 # don't import zlib again.

maubot/loader/meta.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import List
+from typing import List, Optional
 
 from attr import dataclass
 from packaging.version import InvalidVersion, Version
@@ -63,3 +63,7 @@ class PluginMeta(SerializableAttrs):
     extra_files: List[str] = []
     dependencies: List[str] = []
     soft_dependencies: List[str] = []
+
+    @property
+    def database_type_str(self) -> Optional[str]:
+        return self.database_type.value if self.database else None

maubot/loader/zip.py

@@ -31,7 +31,7 @@ from ..config import Config
 from ..lib.zipimport import ZipImportError, zipimporter
 from ..plugin_base import Plugin
 from .abc import IDConflictError, PluginClass, PluginLoader
-from .meta import PluginMeta
+from .meta import DatabaseType, PluginMeta
 
 current_version = Version(__version__)
 yaml = YAML()
@@ -155,9 +155,9 @@ class ZippedPluginLoader(PluginLoader):
         return file, meta
 
     @classmethod
-    def verify_meta(cls, source) -> tuple[str, Version]:
+    def verify_meta(cls, source) -> tuple[str, Version, DatabaseType | None]:
         _, meta = cls._read_meta(source)
-        return meta.id, meta.version
+        return meta.id, meta.version, meta.database_type if meta.database else None
 
     def _load_meta(self) -> None:
         file, meta = self._read_meta(self.path)
@@ -167,7 +167,7 @@ class ZippedPluginLoader(PluginLoader):
         if "/" in meta.main_class:
             self.main_module, self.main_class = meta.main_class.split("/")[:2]
         else:
-            self.main_module = meta.modules[0]
+            self.main_module = meta.modules[-1]
             self.main_class = meta.main_class
         self._file = file
 

maubot/management/api/client.py

@@ -16,6 +16,7 @@
 from __future__ import annotations
 
 from json import JSONDecodeError
+import logging
 
 from aiohttp import web
 
@@ -27,6 +28,8 @@ from ...client import Client
 from .base import routes
 from .responses import resp
 
+log = logging.getLogger("maubot.server.client")
+
 
 @routes.get("/clients")
 async def get_clients(_: web.Request) -> web.Response:
@@ -54,11 +57,13 @@ async def _create_client(user_id: UserID | None, data: dict) -> web.Response:
     )
     try:
         whoami = await new_client.whoami()
-    except MatrixInvalidToken:
+    except MatrixInvalidToken as e:
         return resp.bad_client_access_token
     except MatrixRequestError:
+        log.warning(f"Failed to get whoami from {homeserver} for new client", exc_info=True)
         return resp.bad_client_access_details
     except MatrixConnectionError:
+        log.warning(f"Failed to connect to {homeserver} for new client", exc_info=True)
         return resp.bad_client_connection_details
     if user_id is None:
         existing_client = await Client.get(whoami.user_id)
@@ -73,8 +78,8 @@ async def _create_client(user_id: UserID | None, data: dict) -> web.Response:
     )
     client.enabled = data.get("enabled", True)
     client.sync = data.get("sync", True)
-    client.autojoin = data.get("autojoin", True)
+    await client.update_autojoin(data.get("autojoin", True), save=False)
-    client.online = data.get("online", True)
+    await client.update_online(data.get("online", True), save=False)
     client.displayname = data.get("displayname", "disable")
     client.avatar_url = data.get("avatar_url", "disable")
     await client.update()
@@ -90,8 +95,12 @@ async def _update_client(client: Client, data: dict, is_login: bool = False) ->
     except MatrixInvalidToken:
         return resp.bad_client_access_token
     except MatrixRequestError:
+        log.warning(
+            f"Failed to get whoami from homeserver to update client details", exc_info=True
+        )
         return resp.bad_client_access_details
     except MatrixConnectionError:
+        log.warning(f"Failed to connect to homeserver to update client details", exc_info=True)
         return resp.bad_client_connection_details
     except ValueError as e:
         str_err = str(e)

maubot/management/api/client_auth.py

@@ -184,11 +184,10 @@ async def _do_sso(req: AuthRequestInfo) -> web.Response:
     cfg = get_config()
     public_url = (
         URL(cfg["server.public_url"])
-        / cfg["server.base_path"].lstrip("/")
+        / "_matrix/maubot/v1/client/auth_external_sso/complete"
-        / "client/auth_external_sso/complete"
         / waiter_id
     )
-    sso_url = req.client.api.base_url.with_path(str(Path.login.sso.redirect)).with_query(
+    sso_url = req.client.api.base_url.with_path(str(Path.v3.login.sso.redirect)).with_query(
         {"redirectUrl": str(public_url)}
     )
     sso_waiters[waiter_id] = req, asyncio.get_running_loop().create_future()

maubot/management/api/instance_database.py

@@ -19,12 +19,12 @@ from datetime import datetime
 
 from aiohttp import web
 from asyncpg import PostgresError
-from sqlalchemy import asc, desc, engine, exc
 import aiosqlite
 
 from mautrix.util.async_db import Database
 
 from ...instance import PluginInstance
+from ...lib.optionalalchemy import Engine, IntegrityError, OperationalError, asc, desc
 from .base import routes
 from .responses import resp
 
@@ -56,15 +56,17 @@ async def get_table(request: web.Request) -> web.Response:
     try:
         order = [tuple(order.split(":")) for order in request.query.getall("order")]
         order = [
-            (asc if sort.lower() == "asc" else desc)(table.columns[column])
+            (
-            if sort
+                (asc if sort.lower() == "asc" else desc)(table.columns[column])
-            else table.columns[column]
+                if sort
+                else table.columns[column]
+            )
             for column, sort in order
         ]
     except KeyError:
         order = []
     limit = int(request.query.get("limit", "100"))
-    if isinstance(instance.inst_db, engine.Engine):
+    if isinstance(instance.inst_db, Engine):
         return _execute_query_sqlalchemy(instance, table.select().order_by(*order).limit(limit))
 
 
@@ -82,7 +84,7 @@ async def query(request: web.Request) -> web.Response:
     except KeyError:
         return resp.query_missing
     rows_as_dict = data.get("rows_as_dict", False)
-    if isinstance(instance.inst_db, engine.Engine):
+    if isinstance(instance.inst_db, Engine):
         return _execute_query_sqlalchemy(instance, sql_query, rows_as_dict)
     elif isinstance(instance.inst_db, Database):
         try:
@@ -131,12 +133,12 @@ async def _execute_query_asyncpg(
 def _execute_query_sqlalchemy(
     instance: PluginInstance, sql_query: str, rows_as_dict: bool = False
 ) -> web.Response:
-    assert isinstance(instance.inst_db, engine.Engine)
+    assert isinstance(instance.inst_db, Engine)
     try:
         res = instance.inst_db.execute(sql_query)
-    except exc.IntegrityError as e:
+    except IntegrityError as e:
         return resp.sql_integrity_error(e, sql_query)
-    except exc.OperationalError as e:
+    except OperationalError as e:
         return resp.sql_operational_error(e, sql_query)
     data = {
         "ok": True,

maubot/management/api/log.py

@@ -22,6 +22,8 @@ import logging
 
 from aiohttp import web, web_ws
 
+from mautrix.util import background_task
+
 from .auth import is_valid_token
 from .base import routes
 
@@ -142,7 +144,7 @@ async def log_websocket(request: web.Request) -> web.WebSocketResponse:
             await ws.close(code=4000)
             log.debug(f"Connection from {request.remote} terminated due to no authentication")
 
-    asyncio.create_task(close_if_not_authenticated())
+    background_task.create(close_if_not_authenticated())
 
     try:
         msg: web_ws.WSMessage

maubot/management/api/middleware.py

@@ -29,7 +29,7 @@ log = logging.getLogger("maubot.server")
 
 @web.middleware
 async def auth(request: web.Request, handler: Handler) -> web.Response:
-    subpath = request.path[len(get_config()["server.base_path"]) :]
+    subpath = request.path[len("/_matrix/maubot/v1") :]
     if (
         subpath.startswith("/auth/")
         or subpath.startswith("/client/auth_external_sso/complete/")

maubot/management/api/plugin_upload.py

@@ -23,10 +23,17 @@ import traceback
 from aiohttp import web
 from packaging.version import Version
 
-from ...loader import MaubotZipImportError, PluginLoader, ZippedPluginLoader
+from ...loader import DatabaseType, MaubotZipImportError, PluginLoader, ZippedPluginLoader
 from .base import get_config, routes
 from .responses import resp
 
+try:
+    import sqlalchemy
+
+    has_alchemy = True
+except ImportError:
+    has_alchemy = False
+
 log = logging.getLogger("maubot.server.upload")
 
 
@@ -36,9 +43,11 @@ async def put_plugin(request: web.Request) -> web.Response:
     content = await request.read()
     file = BytesIO(content)
     try:
-        pid, version = ZippedPluginLoader.verify_meta(file)
+        pid, version, db_type = ZippedPluginLoader.verify_meta(file)
     except MaubotZipImportError as e:
         return resp.plugin_import_error(str(e), traceback.format_exc())
+    if db_type == DatabaseType.SQLALCHEMY and not has_alchemy:
+        return resp.sqlalchemy_not_installed
     if pid != plugin_id:
         return resp.pid_mismatch
     plugin = PluginLoader.id_cache.get(plugin_id, None)
@@ -55,9 +64,11 @@ async def upload_plugin(request: web.Request) -> web.Response:
     content = await request.read()
     file = BytesIO(content)
     try:
-        pid, version = ZippedPluginLoader.verify_meta(file)
+        pid, version, db_type = ZippedPluginLoader.verify_meta(file)
     except MaubotZipImportError as e:
         return resp.plugin_import_error(str(e), traceback.format_exc())
+    if db_type == DatabaseType.SQLALCHEMY and not has_alchemy:
+        return resp.sqlalchemy_not_installed
     plugin = PluginLoader.id_cache.get(pid, None)
     if not plugin:
         return await upload_new_plugin(content, pid, version)

maubot/management/api/responses.py

@@ -15,13 +15,16 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
 from http import HTTPStatus
 
 from aiohttp import web
 from asyncpg import PostgresError
-from sqlalchemy.exc import IntegrityError, OperationalError
 import aiosqlite
 
+if TYPE_CHECKING:
+    from sqlalchemy.exc import IntegrityError, OperationalError
+
 
 class _Response:
     @property
@@ -324,6 +327,16 @@ class _Response:
             }
         )
 
+    @property
+    def sqlalchemy_not_installed(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "This plugin requires a legacy database, but SQLAlchemy is not installed",
+                "errcode": "unsupported_plugin_database",
+            },
+            status=HTTPStatus.NOT_IMPLEMENTED,
+        )
+
     @property
     def table_not_found(self) -> web.Response:
         return web.json_response(

maubot/management/frontend/src/api.js

@@ -205,7 +205,7 @@ export const getClients = () => defaultGet("/clients")
 export const getClient = id => defaultGet(`/clients/${id}`)
 
 export async function uploadAvatar(id, data, mime) {
-    const resp = await fetch(`${BASE_PATH}/proxy/${id}/_matrix/media/r0/upload`, {
+    const resp = await fetch(`${BASE_PATH}/proxy/${id}/_matrix/media/v3/upload`, {
         headers: getHeaders(mime),
         body: data,
         method: "POST",
@@ -217,8 +217,9 @@ export function getAvatarURL({ id, avatar_url }) {
     if (!avatar_url?.startsWith("mxc://")) {
         return null
     }
-    avatar_url = avatar_url.substr("mxc://".length)
+    avatar_url = avatar_url.substring("mxc://".length)
-    return `${BASE_PATH}/proxy/${id}/_matrix/media/r0/download/${avatar_url}?access_token=${
+    // Note: the maubot backend will replace the query param with an authorization header
+    return `${BASE_PATH}/proxy/${id}/_matrix/client/v1/media/download/${avatar_url}?access_token=${
         localStorage.accessToken}`
 }
 

maubot/management/frontend/src/pages/Main.js

@@ -45,9 +45,8 @@ class Main extends Component {
             const resp = await fetch(process.env.PUBLIC_URL + "/paths.json", {
                 headers: { "Content-Type": "application/json" },
             })
-            const apiPathJson = await resp.json()
+            const apiPaths = await resp.json()
-            const apiPath = apiPathJson.api_path
+            api.setBasePath(apiPaths.api_path)
-            api.setBasePath(`${apiPath}`)
         } catch (err) {
             console.error("Failed to get API path:", err)
         }

maubot/management/frontend/src/pages/dashboard/Instance.js

@@ -43,7 +43,7 @@ class Instance extends BaseMainView {
     }
 
     get entryKeys() {
-        return ["id", "primary_user", "enabled", "started", "type", "config"]
+        return ["id", "primary_user", "enabled", "started", "type", "config", "database_engine"]
     }
 
     get initialState() {
@@ -54,6 +54,7 @@ class Instance extends BaseMainView {
             started: true,
             type: "",
             config: "",
+            database_engine: "",
 
             saving: false,
             deleting: false,

maubot/management/frontend/src/pages/dashboard/Log.js

@@ -41,7 +41,7 @@ class LogEntry extends PureComponent {
             const req = this.props.line.matrix_http_request
 
             return <>
-                {req.method} {req.path}
+                {req.method} {req.url || req.path}
                 <div className="content">
                     {Object.entries(req.content || {}).length > 0
                     && <JSONTree data={{ content: req.content }} hideRoot={true}/>}

maubot/matrix.py

@@ -24,6 +24,7 @@ import attr
 from mautrix.client import Client as MatrixClient, SyncStream
 from mautrix.errors import DecryptionError
 from mautrix.types import (
+    BaseMessageEventContentFuncs,
     EncryptedEvent,
     Event,
     EventID,
@@ -61,7 +62,10 @@ async def parse_formatted(
         html = message
     else:
         return message, escape(message)
-    return (await MaubotHTMLParser().parse(html)).text, html
+    text = (await MaubotHTMLParser().parse(html)).text
+    if len(text) + len(html) > 40000:
+        text = text[:100] + "[long message cut off]"
+    return text, html
 
 
 class MaubotMessageEvent(MessageEvent):
@@ -82,8 +86,31 @@ class MaubotMessageEvent(MessageEvent):
         markdown: bool = True,
         allow_html: bool = False,
         reply: bool | str = False,
+        in_thread: bool | None = None,
         edits: EventID | MessageEvent | None = None,
     ) -> EventID:
+        """
+        Respond to the message.
+
+        Args:
+            content: The content to respond with. If this is a string, it will be passed to
+                :func:`parse_formatted` with the markdown and allow_html flags.
+                Otherwise, the content is used as-is
+            event_type: The type of event to send.
+            markdown: When content is a string, should it be parsed as markdown?
+            allow_html: When content is a string, should it allow raw HTML?
+            reply: Should the response be sent as a reply to this event?
+            in_thread: Should the response be sent in a thread with this event?
+                By default (``None``), the response will be in a thread if this event is in a
+                thread. If set to ``False``, the response will never be in a thread. If set to
+                ``True``, the response will always be in a thread, creating one with this event as
+                the root if necessary.
+            edits: An event ID or MessageEvent to edit. If set, the reply and in_thread parameters
+                are ignored, as edits can't change the reply or thread status.
+
+        Returns:
+            The ID of the response event.
+        """
         if isinstance(content, str):
             content = TextMessageEventContent(msgtype=MessageType.NOTICE, body=content)
             if allow_html or markdown:
@@ -93,7 +120,19 @@ class MaubotMessageEvent(MessageEvent):
                 )
         if edits:
             content.set_edit(edits)
-        elif reply:
+        if (
+            not edits
+            and in_thread is not False
+            and (
+                in_thread
+                or (
+                    isinstance(self.content, BaseMessageEventContentFuncs)
+                    and self.content.get_thread_parent()
+                )
+            )
+        ):
+            content.set_thread_parent(self)
+        if reply and not edits:
             if reply != "force" and self.disable_reply:
                 content.body = f"{self.sender}: {content.body}"
                 fmt_body = content.formatted_body or escape(content.body).replace("\n", "<br>")
@@ -112,17 +151,71 @@ class MaubotMessageEvent(MessageEvent):
         event_type: EventType = EventType.ROOM_MESSAGE,
         markdown: bool = True,
         allow_html: bool = False,
+        in_thread: bool | None = None,
     ) -> Awaitable[EventID]:
+        """
+        Reply to the message. The parameters are the same as :meth:`respond`,
+        but ``reply`` is always ``True`` and ``edits`` is not supported.
+
+        Args:
+            content: The content to respond with. If this is a string, it will be passed to
+                :func:`parse_formatted` with the markdown and allow_html flags.
+                Otherwise, the content is used as-is
+            event_type: The type of event to send.
+            markdown: When content is a string, should it be parsed as markdown?
+            allow_html: When content is a string, should it allow raw HTML?
+            in_thread: Should the response be sent in a thread with this event?
+                By default (``None``), the response will be in a thread if this event is in a
+                thread. If set to ``False``, the response will never be in a thread. If set to
+                ``True``, the response will always be in a thread, creating one with this event as
+                the root if necessary.
+
+        Returns:
+            The ID of the response event.
+        """
         return self.respond(
-            content, event_type, markdown=markdown, reply=True, allow_html=allow_html
+            content,
+            event_type,
+            markdown=markdown,
+            reply=True,
+            in_thread=in_thread,
+            allow_html=allow_html,
         )
 
     def mark_read(self) -> Awaitable[None]:
+        """
+        Mark this event as read.
+        """
         return self.client.send_receipt(self.room_id, self.event_id, "m.read")
 
     def react(self, key: str) -> Awaitable[EventID]:
+        """
+        React to this event with the given key.
+
+        Args:
+            key: The key to react with. Often an unicode emoji.
+
+        Returns:
+            The ID of the reaction event.
+
+        Examples:
+            >>> evt: MaubotMessageEvent
+            >>> evt.react("🐈️")
+        """
         return self.client.react(self.room_id, self.event_id, key)
 
+    def redact(self, reason: str | None = None) -> Awaitable[EventID]:
+        """
+        Redact this event.
+
+        Args:
+            reason: Optionally, the reason for redacting the event.
+
+        Returns:
+            The ID of the redaction event.
+        """
+        return self.client.redact(self.room_id, self.event_id, reason=reason)
+
     def edit(
         self,
         content: str | MessageEventContent,
@@ -130,6 +223,21 @@ class MaubotMessageEvent(MessageEvent):
         markdown: bool = True,
         allow_html: bool = False,
     ) -> Awaitable[EventID]:
+        """
+        Edit this event. Note that other clients will only render the edit if it was sent by the
+        same user who's doing the editing.
+
+        Args:
+            content: The new content for the event. If this is a string, it will be passed to
+                :func:`parse_formatted` with the markdown and allow_html flags.
+                Otherwise, the content is used as-is.
+            event_type: The type of event to edit into.
+            markdown: When content is a string, should it be parsed as markdown?
+            allow_html: When content is a string, should it allow raw HTML?
+
+        Returns:
+            The ID of the edit event.
+        """
         return self.respond(
             content, event_type, markdown=markdown, edits=self, allow_html=allow_html
         )

maubot/plugin_base.py

@@ -20,14 +20,17 @@ from abc import ABC
 from asyncio import AbstractEventLoop
 
 from aiohttp import ClientSession
-from sqlalchemy.engine.base import Engine
 from yarl import URL
 
 from mautrix.util.async_db import Database, UpgradeTable
 from mautrix.util.config import BaseProxyConfig
 from mautrix.util.logging import TraceLogger
 
+from .scheduler import BasicScheduler
+
 if TYPE_CHECKING:
+    from sqlalchemy.engine.base import Engine
+
     from .client import MaubotMatrixClient
     from .loader import BasePluginLoader
     from .plugin_server import PluginWebApp
@@ -40,6 +43,7 @@ class Plugin(ABC):
     log: TraceLogger
     loop: AbstractEventLoop
     loader: BasePluginLoader
+    sched: BasicScheduler
     config: BaseProxyConfig | None
     database: Engine | Database | None
     webapp: PluginWebApp | None
@@ -53,11 +57,12 @@ class Plugin(ABC):
         instance_id: str,
         log: TraceLogger,
         config: BaseProxyConfig | None,
-        database: Engine | None,
+        database: Engine | Database | None,
         webapp: PluginWebApp | None,
         webapp_url: str | None,
         loader: BasePluginLoader,
     ) -> None:
+        self.sched = BasicScheduler(log=log.getChild("scheduler"))
         self.client = client
         self.loop = loop
         self.http = http
@@ -76,8 +81,9 @@ class Plugin(ABC):
             val = getattr(obj, key)
             try:
                 if val.__mb_event_handler__:
-                    self._handlers_at_startup.append((val, val.__mb_event_type__))
+                    for event_type in val.__mb_event_types__:
-                    self.client.add_event_handler(val.__mb_event_type__, val)
+                        self._handlers_at_startup.append((val, event_type))
+                        self.client.add_event_handler(event_type, val)
             except AttributeError:
                 pass
             try:
@@ -116,6 +122,7 @@ class Plugin(ABC):
             self.client.remove_event_handler(event_type, func)
         if self.webapp is not None:
             self.webapp.clear()
+        self.sched.stop()
         await self.stop()
 
     async def stop(self) -> None:

maubot/plugin_server.py

@@ -40,6 +40,8 @@ class PluginWebApp(web.UrlDispatcher):
         self._resources = []
         self._named_resources = {}
         self._middleware = []
+        self._resource_index = {}
+        self._matched_sub_app_resources = []
 
     async def handle(self, request: web.Request) -> web.StreamResponse:
         match_info = await self.resolve(request)

maubot/scheduler.py

@@ -0,0 +1,159 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2024 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import Awaitable, Callable
+import asyncio
+import logging
+
+
+class BasicScheduler:
+    background_loop: asyncio.Task | None
+    tasks: set[asyncio.Task]
+    log: logging.Logger
+
+    def __init__(self, log: logging.Logger) -> None:
+        self.log = log
+        self.tasks = set()
+
+    def _find_caller(self) -> str:
+        try:
+            file_name, line_number, function_name, _ = self.log.findCaller()
+            return f"{function_name} at {file_name}:{line_number}"
+        except ValueError:
+            return "unknown function"
+
+    def run_periodically(
+        self,
+        period: float | int,
+        func: Callable[[], Awaitable],
+        run_task_in_background: bool = False,
+        catch_errors: bool = True,
+    ) -> asyncio.Task:
+        """
+        Run a function periodically in the background.
+
+        Args:
+            period: The period in seconds between each call to the function.
+            func: The function to run. No parameters will be provided,
+                use :meth:`functools.partial` if you need to pass parameters.
+            run_task_in_background: If ``True``, the function will be run in a background task.
+                If ``False`` (the default), the loop will wait for the task to return before
+                sleeping for the next period.
+            catch_errors: Whether the scheduler should catch and log any errors.
+                If ``False``, errors will be raised, and the caller must await the returned task
+                to find errors. This parameter has no effect if ``run_task_in_background``
+                is ``True``.
+
+        Returns:
+            The asyncio task object representing the background loop.
+        """
+        task = asyncio.create_task(
+            self._call_periodically(
+                period,
+                func,
+                caller=self._find_caller(),
+                catch_errors=catch_errors,
+                run_task_in_background=run_task_in_background,
+            )
+        )
+        self._register_task(task)
+        return task
+
+    def run_later(
+        self, delay: float | int, coro: Awaitable, catch_errors: bool = True
+    ) -> asyncio.Task:
+        """
+        Run a coroutine after a delay.
+
+        Examples:
+            >>> self.sched.run_later(5, self.async_task(meow=True))
+
+        Args:
+            delay: The delay in seconds to await the coroutine after.
+            coro: The coroutine to await.
+            catch_errors: Whether the scheduler should catch and log any errors.
+                If ``False``, errors will be raised, and the caller must await the returned task
+                to find errors.
+
+        Returns:
+            The asyncio task object representing the scheduled task.
+        """
+        task = asyncio.create_task(
+            self._call_with_delay(
+                delay, coro, caller=self._find_caller(), catch_errors=catch_errors
+            )
+        )
+        self._register_task(task)
+        return task
+
+    def _register_task(self, task: asyncio.Task) -> None:
+        self.tasks.add(task)
+        task.add_done_callback(self.tasks.discard)
+
+    async def _call_periodically(
+        self,
+        period: float | int,
+        func: Callable[[], Awaitable],
+        caller: str,
+        catch_errors: bool,
+        run_task_in_background: bool,
+    ) -> None:
+        while True:
+            try:
+                await asyncio.sleep(period)
+                if run_task_in_background:
+                    self._register_task(
+                        asyncio.create_task(self._call_periodically_background(func(), caller))
+                    )
+                else:
+                    await func()
+            except asyncio.CancelledError:
+                raise
+            except Exception:
+                if catch_errors:
+                    self.log.exception(f"Uncaught error in background loop (created in {caller})")
+                else:
+                    raise
+
+    async def _call_periodically_background(self, coro: Awaitable, caller: str) -> None:
+        try:
+            await coro
+        except asyncio.CancelledError:
+            raise
+        except Exception:
+            self.log.exception(f"Uncaught error in background loop subtask (created in {caller})")
+
+    async def _call_with_delay(
+        self, delay: float | int, coro: Awaitable, caller: str, catch_errors: bool
+    ) -> None:
+        try:
+            await asyncio.sleep(delay)
+            await coro
+        except asyncio.CancelledError:
+            raise
+        except Exception:
+            if catch_errors:
+                self.log.exception(f"Uncaught error in scheduled task (created in {caller})")
+            else:
+                raise
+
+    def stop(self) -> None:
+        """
+        Stop all scheduled tasks and background loops.
+        """
+        for task in self.tasks:
+            task.cancel(msg="Scheduler stopped")

maubot/server.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
+from typing import Callable
 import asyncio
 import json
 import logging
@@ -52,7 +53,7 @@ class MaubotServer:
         self.config = config
 
         self.setup_appservice()
-        self.app.add_subapp(config["server.base_path"], management_api)
+        self.app.add_subapp("/_matrix/maubot/v1", management_api)
         self.setup_instance_subapps()
         self.setup_management_ui()
 
@@ -63,14 +64,14 @@ class MaubotServer:
             if request.path.startswith(path):
                 request = request.clone(
                     rel_url=request.rel_url.with_path(
-                        request.rel_url.path[len(path) :]
+                        request.rel_url.path[len(path) - 1 :]
                     ).with_query(request.query_string)
                 )
                 return await app.handle(request)
         return web.Response(status=404)
 
     def get_instance_subapp(self, instance_id: str) -> tuple[PluginWebApp, str]:
-        subpath = self.config["server.plugin_base_path"] + instance_id
+        subpath = self.config["server.plugin_base_path"] + instance_id + "/"
         url = self.config["server.public_url"] + subpath
         try:
             return self.plugin_routes[subpath], url
@@ -81,7 +82,7 @@ class MaubotServer:
 
     def remove_instance_webapp(self, instance_id: str) -> None:
         try:
-            subpath = self.config["server.plugin_base_path"] + instance_id
+            subpath = self.config["server.plugin_base_path"] + instance_id + "/"
             self.plugin_routes.pop(subpath).clear()
         except KeyError:
             return
@@ -93,7 +94,7 @@ class MaubotServer:
         self.app.router.register_resource(resource)
 
     def setup_appservice(self) -> None:
-        as_path = PathBuilder(self.config["server.appservice_base_path"])
+        as_path = PathBuilder("/_matrix/appservice/v1")
         self.add_route(Method.PUT, as_path.transactions, self.handle_transaction)
 
     def setup_management_ui(self) -> None:
@@ -127,6 +128,13 @@ class MaubotServer:
         )
         self.app.router.add_get(ui_base, ui_base_redirect)
 
+    @staticmethod
+    def _static_data(data: bytes, mime: str) -> Callable[[web.Request], web.Response]:
+        def fn(_: web.Request) -> web.Response:
+            return web.Response(body=data, content_type=mime)
+
+        return fn
+
     def setup_static_root_files(self, directory: str, ui_base: str) -> None:
         files = {
             "asset-manifest.json": "application/json",
@@ -136,20 +144,14 @@ class MaubotServer:
         for file, mime in files.items():
             with open(f"{directory}/{file}", "rb") as stream:
                 data = stream.read()
-            self.app.router.add_get(
+            self.app.router.add_get(f"{ui_base}/{file}", self._static_data(data, mime))
-                f"{ui_base}/{file}", lambda _: web.Response(body=data, content_type=mime)
-            )
 
-        # also set up a resource path for the public url path prefix config
-        # cut the prefix path from public_url
         public_url = self.config["server.public_url"]
-        base_path = self.config["server.base_path"]
         public_url_path = ""
         if public_url:
             public_url_path = URL(public_url).path.rstrip("/")
 
-        # assemble with base_path
+        api_path = f"{public_url_path}/_matrix/maubot/v1"
-        api_path = f"{public_url_path}{base_path}"
 
         path_prefix_response_body = json.dumps({"api_path": api_path.rstrip("/")})
         self.app.router.add_get(

maubot/standalone/Dockerfile

@@ -1,9 +1,8 @@
-FROM docker.io/alpine:3.15
+FROM docker.io/alpine:3.21
 
 RUN apk add --no-cache \
       python3 py3-pip py3-setuptools py3-wheel \
       py3-aiohttp \
-      py3-sqlalchemy \
       py3-attrs \
       py3-bcrypt \
       py3-cffi \
@@ -26,8 +25,8 @@ RUN cd /opt/maubot \
       python3-dev \
       libffi-dev \
       build-base \
-  && pip3 install -r requirements.txt -r optional-requirements.txt \
+  && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \
   && apk del .build-deps
 
 COPY . /opt/maubot
-RUN cd /opt/maubot && pip3 install .
+RUN cd /opt/maubot && pip3 install --break-system-packages .

maubot/standalone/__main__.py

@@ -30,7 +30,11 @@ from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap
 from yarl import URL
 
+from mautrix.appservice import AppServiceServerMixin
+from mautrix.client import SyncStream
 from mautrix.types import (
+    BaseMessageEventContentFuncs,
+    Event,
     EventType,
     Filter,
     Membership,
@@ -111,11 +115,14 @@ with open(args.meta, "r") as meta_file:
 if "/" in meta.main_class:
     module, main_class = meta.main_class.split("/", 1)
 else:
-    module = meta.modules[0]
+    module = meta.modules[-1]
     main_class = meta.main_class
+
+if args.meta != "maubot.yaml" and os.path.dirname(args.meta) != "":
+    sys.path.append(os.path.dirname(args.meta))
 bot_module = importlib.import_module(module)
 plugin: type[Plugin] = getattr(bot_module, main_class)
-loader = FileSystemLoader(os.path.dirname(args.meta))
+loader = FileSystemLoader(os.path.dirname(args.meta), meta)
 
 log.info(f"Initializing standalone {meta.id} v{meta.version} on maubot {__version__}")
 
@@ -131,6 +138,7 @@ user_id = config["user.credentials.id"]
 device_id = config["user.credentials.device_id"]
 homeserver = config["user.credentials.homeserver"]
 access_token = config["user.credentials.access_token"]
+appservice_listener = config["user.appservice"]
 
 crypto_store = state_store = None
 if device_id and not OlmMachine:
@@ -188,6 +196,10 @@ if meta.webapp:
     resource = PrefixResource(web_base_path)
     resource.add_route(hdrs.METH_ANY, _handle_plugin_request)
     web_app.router.register_resource(resource)
+elif appservice_listener:
+    web_app = web.Application()
+    web_runner = web.AppRunner(web_app, access_log_class=AccessLogger)
+    public_url = plugin_webapp = None
 else:
     web_app = web_runner = public_url = plugin_webapp = None
 
@@ -195,6 +207,31 @@ loop = asyncio.get_event_loop()
 
 client: MaubotMatrixClient | None = None
 bot: Plugin | None = None
+appservice: AppServiceServerMixin | None = None
+
+
+if appservice_listener:
+    assert web_app is not None, "web_app is always set when appservice_listener is set"
+    appservice = AppServiceServerMixin(
+        ephemeral_events=True,
+        encryption_events=True,
+        log=logging.getLogger("maubot.appservice"),
+        hs_token=config["user.hs_token"],
+    )
+    appservice.register_routes(web_app)
+
+    @appservice.matrix_event_handler
+    async def handle_appservice_event(evt: Event) -> None:
+        if isinstance(evt.content, BaseMessageEventContentFuncs):
+            evt.content.trim_reply_fallback()
+        fake_sync_stream = SyncStream.JOINED_ROOM
+        if evt.type.is_ephemeral:
+            fake_sync_stream |= SyncStream.EPHEMERAL
+        else:
+            fake_sync_stream |= SyncStream.TIMELINE
+        setattr(evt, "source", fake_sync_stream)
+        tasks = client.dispatch_manual_event(evt.type, evt, include_global_handlers=True)
+        await asyncio.gather(*tasks)
 
 
 async def main():
@@ -217,6 +254,8 @@ async def main():
         state_store=state_store,
         device_id=device_id,
     )
+    if appservice:
+        client.api.as_user_id = user_id
     client.ignore_first_sync = config["user.ignore_first_sync"]
     client.ignore_initial_sync = config["user.ignore_initial_sync"]
     if crypto_store:
@@ -225,6 +264,11 @@ async def main():
         await crypto_store.open()
 
         client.crypto = OlmMachine(client, crypto_store, state_store)
+        if appservice:
+            appservice.otk_handler = client.crypto.handle_as_otk_counts
+            appservice.device_list_handler = client.crypto.handle_as_device_lists
+            appservice.to_device_handler = client.crypto.handle_as_to_device_event
+            client.api.as_device_id = device_id
         crypto_device_id = await crypto_store.get_device_id()
         if crypto_device_id and crypto_device_id != device_id:
             log.fatal(
@@ -272,6 +316,8 @@ async def main():
             )
             await nb.put_filter_id(filter_id)
         _ = client.start(nb.filter_id)
+    elif appservice_listener and crypto_store and not client.crypto.account.shared:
+        await client.crypto.share_keys()
 
     if config["user.autojoin"]:
         log.debug("Autojoin is enabled")
@@ -334,9 +380,14 @@ async def stop(suppress_stop_error: bool = False) -> None:
         except Exception:
             if not suppress_stop_error:
                 log.exception("Error stopping bot")
-    if web_runner:
+    if web_runner and web_runner.server:
-        await web_runner.shutdown()
+        try:
-        await web_runner.cleanup()
+            await web_runner.shutdown()
+            await web_runner.cleanup()
+        except RuntimeError:
+            if not suppress_stop_error:
+                await db.stop()
+                raise
     await db.stop()
 
 
@@ -347,6 +398,10 @@ signal.signal(signal.SIGTERM, signal.default_int_handler)
 try:
     log.info("Starting plugin")
     loop.run_until_complete(main())
+except SystemExit:
+    loop.run_until_complete(stop(suppress_stop_error=True))
+    loop.close()
+    raise
 except (Exception, KeyboardInterrupt) as e:
     if isinstance(e, KeyboardInterrupt):
         log.info("Startup interrupted, stopping")

maubot/standalone/config.py

@@ -33,9 +33,13 @@ class Config(BaseFileConfig):
         copy("user.credentials.access_token")
         copy("user.credentials.device_id")
         copy("user.sync")
+        copy("user.appservice")
+        copy("user.hs_token")
         copy("user.autojoin")
         copy("user.displayname")
         copy("user.avatar_url")
+        copy("user.ignore_initial_sync")
+        copy("user.ignore_first_sync")
         if "server" in base:
             copy("server.hostname")
             copy("server.port")

maubot/standalone/example-config.yaml

@@ -5,9 +5,15 @@ user:
         homeserver: https://example.com
         access_token: foo
         # If you want to enable encryption, set the device ID corresponding to the access token here.
+        # When using an appservice, you should use appservice login manually to generate a device ID and access token.
         device_id: null
     # Enable /sync? This is not needed for purely unencrypted webhook-based bots, but is necessary in most other cases.
     sync: true
+    # Receive appservice transactions? This will add a /_matrix/app/v1/transactions endpoint on
+    # the HTTP server configured below. The base_path will not be applied for the /transactions path.
+    appservice: false
+    # When appservice mode is enabled, the hs_token for the appservice.
+    hs_token: null
     # Automatically accept invites?
     autojoin: false
     # The displayname and avatar URL to set for the bot on startup.
@@ -21,7 +27,8 @@ user:
     # if you want the bot to handle messages that were sent while the bot was down.
     ignore_first_sync: true
 
-# Web server settings. These will only take effect if the plugin requests it using `webapp: true` in the meta file.
+# Web server settings. These will only take effect if the plugin requests it using `webapp: true` in the meta file,
+# or if user -> appservice is set to true.
 server:
     # The IP and port to listen to.
     hostname: 0.0.0.0
@@ -35,7 +42,7 @@ server:
 
 # The database for the plugin. Used for plugin data, the sync token and e2ee data (if enabled).
 # SQLite and Postgres are supported.
-database: sqlite:///bot.db
+database: sqlite:bot.db
 
 # Additional arguments for asyncpg.create_pool() or sqlite3.connect()
 # https://magicstack.github.io/asyncpg/current/api/index.html#asyncpg.pool.create_pool

maubot/standalone/loader.py

@@ -18,12 +18,13 @@ from __future__ import annotations
 import os
 import os.path
 
-from ..loader import BasePluginLoader
+from ..loader import BasePluginLoader, PluginMeta
 
 
 class FileSystemLoader(BasePluginLoader):
-    def __init__(self, path: str) -> None:
+    def __init__(self, path: str, meta: PluginMeta) -> None:
         self.path = path
+        self.meta = meta
 
     @property
     def source(self) -> str:

maubot/testing/__init__.py

@@ -0,0 +1,17 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2023 Aurélien Bompard
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from .bot import TestBot, make_message  # noqa: F401
+from .fixtures import *  # noqa: F401,F403

maubot/testing/bot.py

@@ -0,0 +1,100 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2023 Aurélien Bompard
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import asyncio
+import time
+
+from attr import dataclass
+
+from maubot.matrix import MaubotMatrixClient, MaubotMessageEvent
+from mautrix.api import HTTPAPI
+from mautrix.types import (
+    EventContent,
+    EventType,
+    MessageEvent,
+    MessageType,
+    RoomID,
+    TextMessageEventContent,
+)
+
+
+@dataclass
+class MatrixEvent:
+    room_id: RoomID
+    event_type: EventType
+    content: EventContent
+    kwargs: dict
+
+
+class TestBot:
+    """A mocked bot used for testing purposes.
+
+    Send messages to the mock Matrix server with the ``send()`` method.
+    Look into the ``responded`` list to get what server has replied.
+    """
+
+    def __init__(self, mxid="@botname:example.com", mxurl="http://matrix.example.com"):
+        api = HTTPAPI(base_url=mxurl)
+        self.client = MaubotMatrixClient(api=api)
+        self.responded = []
+        self.client.mxid = mxid
+        self.client.send_message_event = self._mock_send_message_event
+
+    async def _mock_send_message_event(self, room_id, event_type, content, txn_id=None, **kwargs):
+        self.responded.append(
+            MatrixEvent(room_id=room_id, event_type=event_type, content=content, kwargs=kwargs)
+        )
+
+    async def dispatch(self, event_type: EventType, event):
+        tasks = self.client.dispatch_manual_event(event_type, event, force_synchronous=True)
+        return await asyncio.gather(*tasks)
+
+    async def send(
+        self,
+        content,
+        html=None,
+        room_id="testroom",
+        msg_type=MessageType.TEXT,
+        sender="@dummy:example.com",
+        timestamp=None,
+    ):
+        event = make_message(
+            content,
+            html=html,
+            room_id=room_id,
+            msg_type=msg_type,
+            sender=sender,
+            timestamp=timestamp,
+        )
+        await self.dispatch(EventType.ROOM_MESSAGE, MaubotMessageEvent(event, self.client))
+
+
+def make_message(
+    content,
+    html=None,
+    room_id="testroom",
+    msg_type=MessageType.TEXT,
+    sender="@dummy:example.com",
+    timestamp=None,
+):
+    """Make a Matrix message event."""
+    return MessageEvent(
+        type=EventType.ROOM_MESSAGE,
+        room_id=room_id,
+        event_id="test",
+        sender=sender,
+        timestamp=timestamp or int(time.time() * 1000),
+        content=TextMessageEventContent(msgtype=msg_type, body=content, formatted_body=html),
+    )

maubot/testing/fixtures.py

@@ -0,0 +1,135 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2023 Aurélien Bompard
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from pathlib import Path
+import asyncio
+import logging
+
+from ruamel.yaml import YAML
+import aiohttp
+import pytest
+import pytest_asyncio
+
+from maubot import Plugin
+from maubot.loader import PluginMeta
+from maubot.standalone.loader import FileSystemLoader
+from mautrix.util.async_db import Database
+from mautrix.util.config import BaseProxyConfig, RecursiveDict
+from mautrix.util.logging import TraceLogger
+
+from .bot import TestBot
+
+
+@pytest_asyncio.fixture
+async def maubot_test_bot():
+    return TestBot()
+
+
+@pytest.fixture
+def maubot_upgrade_table():
+    return None
+
+
+@pytest.fixture
+def maubot_plugin_path():
+    return Path(".")
+
+
+@pytest.fixture
+def maubot_plugin_meta(maubot_plugin_path):
+    yaml = YAML()
+    with open(maubot_plugin_path.joinpath("maubot.yaml")) as fh:
+        plugin_meta = PluginMeta.deserialize(yaml.load(fh.read()))
+    return plugin_meta
+
+
+@pytest_asyncio.fixture
+async def maubot_plugin_db(tmp_path, maubot_plugin_meta, maubot_upgrade_table):
+    if not maubot_plugin_meta.get("database", False):
+        return
+    db_path = tmp_path.joinpath("maubot-tests.db").as_posix()
+    db = Database.create(
+        f"sqlite:{db_path}",
+        upgrade_table=maubot_upgrade_table,
+        log=logging.getLogger("db"),
+    )
+    await db.start()
+    yield db
+    await db.stop()
+
+
+@pytest.fixture
+def maubot_plugin_class():
+    return Plugin
+
+
+@pytest.fixture
+def maubot_plugin_config_class():
+    return BaseProxyConfig
+
+
+@pytest.fixture
+def maubot_plugin_config_dict():
+    return {}
+
+
+@pytest.fixture
+def maubot_plugin_config_overrides():
+    return {}
+
+
+@pytest.fixture
+def maubot_plugin_config(
+    maubot_plugin_path,
+    maubot_plugin_config_class,
+    maubot_plugin_config_dict,
+    maubot_plugin_config_overrides,
+):
+    yaml = YAML()
+    with open(maubot_plugin_path.joinpath("base-config.yaml")) as fh:
+        base_config = RecursiveDict(yaml.load(fh))
+    maubot_plugin_config_dict.update(maubot_plugin_config_overrides)
+    return maubot_plugin_config_class(
+        load=lambda: maubot_plugin_config_dict,
+        load_base=lambda: base_config,
+        save=lambda c: None,
+    )
+
+
+@pytest_asyncio.fixture
+async def maubot_plugin(
+    maubot_test_bot,
+    maubot_plugin_db,
+    maubot_plugin_class,
+    maubot_plugin_path,
+    maubot_plugin_config,
+    maubot_plugin_meta,
+):
+    loader = FileSystemLoader(maubot_plugin_path, maubot_plugin_meta)
+    async with aiohttp.ClientSession() as http:
+        instance = maubot_plugin_class(
+            client=maubot_test_bot.client,
+            loop=asyncio.get_running_loop(),
+            http=http,
+            instance_id="tests",
+            log=TraceLogger("test"),
+            config=maubot_plugin_config,
+            database=maubot_plugin_db,
+            webapp=None,
+            webapp_url=None,
+            loader=loader,
+        )
+        await instance.internal_start()
+        yield instance

optional-requirements.txt

@@ -5,3 +5,10 @@
 python-olm>=3,<4
 pycryptodome>=3,<4
 unpaddedbase64>=1,<3
+
+#/testing
+pytest
+pytest-asyncio
+
+#/legacydb
+SQLAlchemy>1,<1.4

pyproject.toml

@@ -9,5 +9,5 @@ skip = ["maubot/management/frontend"]
 
 [tool.black]
 line-length = 99
-target-version = ["py38"]
+target-version = ["py310"]
 force-exclude = "maubot/management/frontend"

requirements.txt

@@ -1,16 +1,16 @@
-mautrix>=0.15.5,<0.16
+mautrix>=0.20.7,<0.21
 aiohttp>=3,<4
 yarl>=1,<2
-SQLAlchemy>=1,<1.4
+asyncpg>=0.20,<1
-asyncpg>=0.20,<0.26
+aiosqlite>=0.16,<1
-aiosqlite>=0.16,<0.18
 commonmark>=0.9,<1
-ruamel.yaml>=0.15.35,<0.18
+ruamel.yaml>=0.15.35,<0.19
 attrs>=18.1.0
-bcrypt>=3,<4
+bcrypt>=3,<5
 packaging>=10
 
 click>=7,<9
 colorama>=0.4,<0.5
-questionary>=1,<2
+questionary>=1,<3
 jinja2>=2,<4
+setuptools

setup.py

@@ -41,7 +41,7 @@ setuptools.setup(
 
     install_requires=install_requires,
     extras_require=extras_require,
-    python_requires="~=3.8",
+    python_requires="~=3.10",
 
     classifiers=[
         "Development Status :: 4 - Beta",
@@ -50,13 +50,16 @@ setuptools.setup(
         "Framework :: AsyncIO",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
     ],
     entry_points="""
         [console_scripts]
         mbc=maubot.cli:app
+        [pytest11]
+        maubot=maubot.testing
     """,
     data_files=[
         (".", ["maubot/example-config.yaml"]),
@@ -68,6 +71,7 @@ setuptools.setup(
             "management/frontend/build/static/css/*",
             "management/frontend/build/static/js/*",
             "management/frontend/build/static/media/*",
+            "py.typed",
         ],
         "maubot.cli": ["res/*"],
         "maubot.standalone": ["example-config.yaml"],