diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index 28d6df2..05a8bd2 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -6,17 +6,16 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 with: - python-version: "3.13" + python-version: "3.10" - uses: isort/isort-action@master with: sortPaths: "./maubot" - uses: psf/black@stable with: src: "./maubot" - version: "24.10.0" - name: pre-commit run: | pip install pre-commit diff --git a/.gitignore b/.gitignore index 9fd28ef..99c9d7d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,13 +7,11 @@ pip-selfcheck.json *.pyc __pycache__ -*.db* -*.log +*.db /*.yaml !example-config.yaml !.pre-commit-config.yaml -/start logs/ plugins/ trash/ diff --git a/.gitlab-ci-plugin.yml b/.gitlab-ci-plugin.yml deleted file mode 100644 index 45ef06b..0000000 --- a/.gitlab-ci-plugin.yml +++ /dev/null @@ -1,29 +0,0 @@ -image: dock.mau.dev/maubot/maubot - -stages: -- build - -variables: - PYTHONPATH: /opt/maubot - -build: - stage: build - except: - - tags - script: - - python3 -m maubot.cli build -o xyz.maubot.$CI_PROJECT_NAME-$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA.mbp - artifacts: - paths: - - "*.mbp" - expire_in: 365 days - -build tags: - stage: build - only: - - tags - script: - - python3 -m maubot.cli build -o xyz.maubot.$CI_PROJECT_NAME-$CI_COMMIT_TAG.mbp - artifacts: - paths: - - "*.mbp" - expire_in: never diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 50d0c15..797e095 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,7 @@ default: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY build frontend: - image: node:22-alpine + image: node:16-alpine stage: build frontend before_script: [] variables: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4a6328e..e1e07d5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v4.1.0 hooks: - id: trailing-whitespace exclude_types: [markdown] @@ -8,13 +8,13 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 24.10.0 + rev: 22.3.0 hooks: - id: black language_version: python3 files: ^maubot/.*\.pyi?$ - repo: https://github.com/PyCQA/isort - rev: 5.13.2 + rev: 5.10.1 hooks: - id: isort - files: ^maubot/.*\.pyi?$ + files: ^maubot/.*\.pyi$ diff --git a/CHANGELOG.md b/CHANGELOG.md index d9de2b7..52d4044 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,84 +1,3 @@ -# v0.5.2 (2025-05-05) - -* Improved tombstone handling to ensure that the tombstone sender has - permissions to invite users to the target room. -* Fixed autojoin and online flags not being applied if set during client - creation (thanks to [@bnsh] in [#258]). -* Fixed plugin web apps not being cleared properly when unloading plugins. - -[@bnsh]: https://github.com/bnsh -[#258]: https://github.com/maubot/maubot/pull/258 - -# v0.5.1 (2025-01-03) - -* Updated Docker image to Alpine 3.21. -* Updated media upload/download endpoints in management frontend - (thanks to [@domrim] in [#253]). -* Fixed plugin web app base path not including a trailing slash - (thanks to [@jkhsjdhjs] in [#240]). -* Changed markdown parsing to cut off plaintext body if necessary to allow - longer formatted messages. -* Updated dependencies to fix Python 3.13 compatibility. - -[@domrim]: https://github.com/domrim -[@jkhsjdhjs]: https://github.com/jkhsjdhjs -[#253]: https://github.com/maubot/maubot/pull/253 -[#240]: https://github.com/maubot/maubot/pull/240 - -# v0.5.0 (2024-08-24) - -* Dropped Python 3.9 support. -* Updated Docker image to Alpine 3.20. -* Updated mautrix-python to 0.20.6 to support authenticated media. -* Removed hard dependency on SQLAlchemy. -* Fixed `main_class` to default to being loaded from the last module instead of - the first if a module name is not explicitly specified. - * This was already the [documented behavior](https://docs.mau.fi/maubot/dev/reference/plugin-metadata.html), - and loading from the first module doesn't make sense due to import order. -* Added simple scheduler utility for running background tasks periodically or - after a certain delay. -* Added testing framework for plugins (thanks to [@abompard] in [#225]). -* Changed `mbc build` to ignore directories declared in `modules` that are - missing an `__init__.py` file. - * Importing the modules at runtime would fail and break the plugin. - To include non-code resources outside modules in the mbp archive, - use `extra_files` instead. - -[#225]: https://github.com/maubot/maubot/issues/225 -[@abompard]: https://github.com/abompard - -# v0.4.2 (2023-09-20) - -* Updated Pillow to 10.0.1. -* Updated Docker image to Alpine 3.18. -* Added logging for errors for /whoami errors when adding new bot accounts. -* Added support for using appservice tokens (including appservice encryption) - in standalone mode. - -# v0.4.1 (2023-03-15) - -* Added `in_thread` parameter to `evt.reply()` and `evt.respond()`. - * By default, responses will go to the thread if the command is in a thread. - * By setting the flag to `True` or `False`, the plugin can force the response - to either be or not be in a thread. -* Fixed static files like the frontend app manifest not being served correctly. -* Fixed `self.loader.meta` not being available to plugins in standalone mode. -* Updated to mautrix-python v0.19.6. - -# v0.4.0 (2023-01-29) - -* Dropped support for using a custom maubot API base path. - * The public URL can still have a path prefix, e.g. when using a reverse - proxy. Both the web interface and `mbc` CLI tool should work fine with - custom prefixes. -* Added `evt.redact()` as a shortcut for `self.client.redact(evt.room_id, evt.event_id)`. -* Fixed `mbc logs` command not working on Python 3.8+. -* Fixed saving plugin configs (broke in v0.3.0). -* Fixed SSO login using the wrong API path (probably broke in v0.3.0). -* Stopped using `cd` in the docker image's `mbc` wrapper to enable using - path-dependent commands like `mbc build` by mounting a directory. -* Updated Docker image to Alpine 3.17. - # v0.3.1 (2022-03-29) * Added encryption dependencies to standalone dockerfile. diff --git a/Dockerfile b/Dockerfile index 2c6bad4..8b7eae7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ -FROM node:22 AS frontend-builder +FROM node:16 AS frontend-builder COPY ./maubot/management/frontend /frontend RUN cd /frontend && yarn --prod && yarn build -FROM alpine:3.21 +FROM alpine:3.15 RUN apk add --no-cache \ python3 py3-pip py3-setuptools py3-wheel \ @@ -11,6 +11,7 @@ RUN apk add --no-cache \ su-exec \ yq \ py3-aiohttp \ + py3-sqlalchemy \ py3-attrs \ py3-bcrypt \ py3-cffi \ @@ -20,10 +21,11 @@ RUN apk add --no-cache \ py3-packaging \ py3-markdown \ py3-alembic \ - py3-cssselect \ +# py3-cssselect \ py3-commonmark \ py3-pygments \ py3-tz \ +# py3-tzlocal \ py3-regex \ py3-wcwidth \ # encryption @@ -37,16 +39,17 @@ RUN apk add --no-cache \ py3-magic \ py3-feedparser \ py3-dateutil \ - py3-lxml \ - py3-semver + py3-lxml +# py3-gitlab +# py3-semver@edge # TODO remove pillow, magic, feedparser, lxml, gitlab and semver when maubot supports installing dependencies COPY requirements.txt /opt/maubot/requirements.txt COPY optional-requirements.txt /opt/maubot/optional-requirements.txt WORKDIR /opt/maubot RUN apk add --virtual .build-deps python3-dev build-base git \ - && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \ - dateparser langdetect python-gitlab pyquery tzlocal \ + && pip3 install -r requirements.txt -r optional-requirements.txt \ + dateparser langdetect python-gitlab pyquery cchardet semver tzlocal cssselect \ && apk del .build-deps # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies diff --git a/Dockerfile.ci b/Dockerfile.ci index 9712a16..8c0f49b 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM alpine:3.21 +FROM alpine:3.15 RUN apk add --no-cache \ python3 py3-pip py3-setuptools py3-wheel \ @@ -6,6 +6,7 @@ RUN apk add --no-cache \ su-exec \ yq \ py3-aiohttp \ + py3-sqlalchemy \ py3-attrs \ py3-bcrypt \ py3-cffi \ @@ -41,8 +42,8 @@ COPY requirements.txt /opt/maubot/requirements.txt COPY optional-requirements.txt /opt/maubot/optional-requirements.txt WORKDIR /opt/maubot RUN apk add --virtual .build-deps python3-dev build-base git \ - && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \ - dateparser langdetect python-gitlab pyquery semver tzlocal cssselect \ + && pip3 install -r requirements.txt -r optional-requirements.txt \ + dateparser langdetect python-gitlab pyquery cchardet semver tzlocal cssselect \ && apk del .build-deps # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies diff --git a/Dockerfile.local b/Dockerfile.local deleted file mode 100644 index d37e220..0000000 --- a/Dockerfile.local +++ /dev/null @@ -1,29 +0,0 @@ -FROM r.batts.cloud/nodejs:22 AS frontend-builder - -COPY ./maubot/management/frontend /frontend -RUN cd /frontend && yarn --prod && yarn build - -FROM r.batts.cloud/debian:bookworm - -RUN apt update && \ - apt install -y --no-install-recommends python3 python3-dev python3-venv python3-semver git gosu yq brotli && \ - apt clean -y && \ - rm -rf /var/lib/apt/lists/* - -COPY requirements.txt /opt/maubot/requirements.txt -COPY optional-requirements.txt /opt/maubot/optional-requirements.txt -WORKDIR /opt/maubot -RUN python3 -m venv /venv \ - && bash -c 'source /venv/bin/activate \ - && pip3 install -r requirements.txt -r optional-requirements.txt \ - dateparser langdetect python-gitlab pyquery tzlocal pyfiglet emoji feedparser brotli' -# TODO also remove pyfiglet, emoji, dateparser, langdetect and pyquery when maubot supports installing dependencies - -COPY . /opt/maubot -RUN cp maubot/example-config.yaml . -COPY ./docker/mbc.sh /usr/local/bin/mbc -COPY --from=frontend-builder /frontend/build /opt/maubot/frontend -ENV UID=1337 GID=1337 XDG_CONFIG_HOME=/data -VOLUME /data - -CMD ["/opt/maubot/docker/run.sh"] diff --git a/README.md b/README.md index 02a4b6f..84317d1 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,56 @@ All setup and usage instructions are located on Matrix room: [#maubot:maunium.net](https://matrix.to/#/#maubot:maunium.net) ## Plugins -A list of plugins can be found at [plugins.mau.bot](https://plugins.mau.bot/). - -To add your plugin to the list, send a pull request to . +Open a pull request or join the Matrix room linked above to get your plugin listed here. The plugin wishlist lives at . + +### Official plugins +* [sed](https://github.com/maubot/sed) - A bot to do sed-like replacements. +* [factorial](https://github.com/maubot/factorial) - A bot to calculate unexpected factorials. +* [media](https://github.com/maubot/media) - A bot that replies with the MXC URI of images you send it. +* [dice](https://github.com/maubot/dice) - A combined dice rolling and calculator bot. +* [karma](https://github.com/maubot/karma) - A user karma tracker bot. +* [xkcd](https://github.com/maubot/xkcd) - A bot to view xkcd comics. +* [echo](https://github.com/maubot/echo) - A bot that echoes pings and other stuff. +* [rss](https://github.com/maubot/rss) - A bot that posts RSS feed updates to Matrix. +* [reminder](https://github.com/maubot/reminder) - A bot to remind you about things. +* [translate](https://github.com/maubot/translate) - A bot to translate words. +* [reactbot](https://github.com/maubot/reactbot) - A bot that responds to messages that match predefined rules. +* [exec](https://github.com/maubot/exec) - A bot that executes code. +* [commitstrip](https://github.com/maubot/commitstrip) - A bot to view CommitStrips. +* [supportportal](https://github.com/maubot/supportportal) - A bot to manage customer support on Matrix. +* †[gitlab](https://github.com/maubot/gitlab) - A GitLab client and webhook receiver. +* [github](https://github.com/maubot/github) - A GitHub client and webhook receiver. +* [tex](https://github.com/maubot/tex) - A bot that renders LaTeX. +* [altalias](https://github.com/maubot/altalias) - A bot that lets users publish alternate aliases in rooms. +* [satwcomic](https://github.com/maubot/satwcomic) - A bot to view SatWComics. +* [songwhip](https://github.com/maubot/songwhip) - A bot to post Songwhip links. +* [manhole](https://github.com/maubot/manhole) - A plugin that lets you access a Python shell inside maubot. + +### 3rd party plugins +* [subreddit linkifier](https://github.com/TomCasavant/RedditMaubot) - A bot that condescendingly corrects a user when they enter an r/subreddit without providing a link to that subreddit +* [giphy](https://github.com/TomCasavant/GiphyMaubot) - A bot that generates a gif (from giphy) given search terms +* [trump](https://github.com/jeffcasavant/MaubotTrumpTweet) - A bot that generates a Trump tweet with the given content +* [poll](https://github.com/TomCasavant/PollMaubot) - A bot that will create a simple poll for users in a room +* [urban](https://github.com/dvdgsng/UrbanMaubot) - A bot that fetches definitions from [Urban Dictionary](https://www.urbandictionary.com/). +* [twilio](https://github.com/jeffcasavant/MaubotTwilio) - Maubot-based SMS bridge +* [tmdb](https://codeberg.org/lomion/tmdb-bot) - A bot that posts information about movies fetched from TheMovieDB.org. +* [invite](https://github.com/williamkray/maubot-invite) - A bot to generate invitation tokens from [matrix-registration](https://github.com/ZerataX/matrix-registration) +* [wolframalpha](https://github.com/ggogel/WolframAlphaMaubot) - A bot that allows requesting information from [WolframAlpha](https://www.wolframalpha.com/). +* †[pingcheck](https://edugit.org/nik/maubot-pingcheck) - A bot to ping the echo bot and send rtt to Icinga passive check +* [ticker](https://github.com/williamkray/maubot-ticker) - A bot to return financial data about a stock or cryptocurrency. +* [weather](https://github.com/kellya/maubot-weather) - A bot to get the weather from wttr.in and return a single line of text for the location specified +* †[youtube previewer](https://github.com/ggogel/YoutubePreviewMaubot) - A bot that responds to a YouTube link with the video title and thumbnail. +* †[reddit previewer](https://github.com/ggogel/RedditPreviewMaubot) - A bot that responds to a link of a reddit post with the sub name and title. If available, uploads the image or video. +* [pocket](https://github.com/jaywink/maubot-pocket) - A bot integrating with Pocket to fetch articles and archive them. +* [alternatingcaps](https://github.com/rom4nik/maubot-alternatingcaps) - A bot repeating last message using aLtErNaTiNg cApS. +* [metric](https://github.com/edwardsdean/maubot_metric_bot) - A bot that will reply to a message that contains imperial units and replace them with metric units. +* [urlpreview](https://github.com/coffeebank/coffee-maubot/tree/master/urlpreview) - A bot that responds to links with a link preview embed, using Matrix API to fetch meta tags + +† Uses a synchronous library which can block the whole maubot process (e.g. requests instead of aiohttp) + +### Deprecated/unmaintained plugins +* [jesaribot](https://github.com/maubot/jesaribot) - A simple bot that replies with an image when you say "jesari". + * Superseded by reactbot +* [gitea](https://github.com/saces/maugitea) - A Gitea client and webhook receiver. diff --git a/dev-requirements.txt b/dev-requirements.txt index bb8c2a0..16231f3 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,3 @@ pre-commit>=2.10.1,<3 isort>=5.10.1,<6 -black>=24,<25 +black>=22.3.0,<22 diff --git a/docker/mbc.sh b/docker/mbc.sh index 5bde65a..bffbd5e 100755 --- a/docker/mbc.sh +++ b/docker/mbc.sh @@ -1,3 +1,3 @@ #!/bin/sh -export PYTHONPATH=/opt/maubot +cd /opt/maubot python3 -m maubot.cli "$@" diff --git a/docker/run.sh b/docker/run.sh index 1ec95a2..2cd9b45 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh function fixperms { chown -R $UID:$GID /var/log /data @@ -13,7 +13,7 @@ function fixdefault { function fixconfig { # Change relative default paths to absolute paths in /data - fixdefault '.database' 'sqlite:maubot.db' 'sqlite:/data/maubot.db' + fixdefault '.database' 'sqlite:///maubot.db' 'sqlite:////data/maubot.db' fixdefault '.plugin_directories.upload' './plugins' '/data/plugins' fixdefault '.plugin_directories.load[0]' './plugins' '/data/plugins' fixdefault '.plugin_directories.trash' './trash' '/data/trash' @@ -30,6 +30,7 @@ mkdir -p /var/log/maubot /data/plugins /data/trash /data/dbs if [ ! -f /data/config.yaml ]; then cp example-config.yaml /data/config.yaml + # Apply some docker-specific adjustments to the config echo "Config file not found. Example config copied to /data/config.yaml" echo "Please modify the config file to your liking and restart the container." fixperms @@ -37,14 +38,11 @@ if [ ! -f /data/config.yaml ]; then exit fi +alembic -x config=/data/config.yaml upgrade head fixperms fixconfig if ls /data/plugins/*.db > /dev/null 2>&1; then mv -n /data/plugins/*.db /data/dbs/ fi -if [ -f "/venv/bin/activate" ] ; then - exec gosu $UID:$GID bash -c 'source /venv/bin/activate && python3 -m maubot -c /data/config.yaml' -fi exec su-exec $UID:$GID python3 -m maubot -c /data/config.yaml - diff --git a/maubot/__meta__.py b/maubot/__meta__.py index 7225152..260c070 100644 --- a/maubot/__meta__.py +++ b/maubot/__meta__.py @@ -1 +1 @@ -__version__ = "0.5.2" +__version__ = "0.3.1" diff --git a/maubot/cli/commands/build.py b/maubot/cli/commands/build.py index 39eca53..ec3ac26 100644 --- a/maubot/cli/commands/build.py +++ b/maubot/cli/commands/build.py @@ -93,16 +93,10 @@ def write_plugin(meta: PluginMeta, output: str | IO) -> None: if os.path.isfile(f"{module}.py"): zip.write(f"{module}.py") elif module is not None and os.path.isdir(module): - if os.path.isfile(f"{module}/__init__.py"): - zipdir(zip, module) - else: - print( - Fore.YELLOW - + f"Module {module} is missing __init__.py, skipping" - + Fore.RESET - ) + zipdir(zip, module) else: print(Fore.YELLOW + f"Module {module} not found, skipping" + Fore.RESET) + for pattern in meta.extra_files: for file in glob.iglob(pattern): zip.write(file) diff --git a/maubot/cli/commands/logs.py b/maubot/cli/commands/logs.py index e0ed07d..9a9c644 100644 --- a/maubot/cli/commands/logs.py +++ b/maubot/cli/commands/logs.py @@ -38,7 +38,13 @@ def logs(server: str, tail: int) -> None: global history_count history_count = tail loop = asyncio.get_event_loop() - loop.run_until_complete(view_logs(server, token)) + future = asyncio.create_task(view_logs(server, token), loop=loop) + try: + loop.run_until_complete(future) + except KeyboardInterrupt: + future.cancel() + loop.run_until_complete(future) + loop.close() def parsedate(entry: Obj) -> None: diff --git a/maubot/cli/config.py b/maubot/cli/config.py index 5fdc4ea..a1adc2f 100644 --- a/maubot/cli/config.py +++ b/maubot/cli/config.py @@ -36,7 +36,6 @@ def get_default_server() -> tuple[str | None, str | None]: server = None if server is None: print(f"{Fore.RED}Default server not configured.{Fore.RESET}") - print(f"Perhaps you forgot to {Fore.CYAN}mbc login{Fore.RESET}?") return None, None return server, _get_token(server) diff --git a/maubot/cli/util/spdx.py b/maubot/cli/util/spdx.py index 69f58b7..10508b3 100644 --- a/maubot/cli/util/spdx.py +++ b/maubot/cli/util/spdx.py @@ -36,10 +36,10 @@ def load() -> None: def get(id: str) -> dict[str, str]: if not spdx_list: load() - return spdx_list[id] + return spdx_list[id.lower()] def valid(id: str) -> bool: if not spdx_list: load() - return id in spdx_list + return id.lower() in spdx_list diff --git a/maubot/client.py b/maubot/client.py index b0fde73..81d3d17 100644 --- a/maubot/client.py +++ b/maubot/client.py @@ -15,7 +15,7 @@ # along with this program. If not, see . from __future__ import annotations -from typing import TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable, cast +from typing import TYPE_CHECKING, Any, AsyncGenerator, AsyncIterable, Awaitable, Callable, cast from collections import defaultdict import asyncio import logging @@ -41,7 +41,6 @@ from mautrix.types import ( SyncToken, UserID, ) -from mautrix.util import background_task from mautrix.util.async_getter_lock import async_getter_lock from mautrix.util.logging import TraceLogger @@ -255,7 +254,7 @@ class Client(DBClient): self.log.warning( f"Failed to get /account/whoami, retrying in {(try_n + 1) * 10}s: {e}" ) - background_task.create(self.start(try_n + 1)) + _ = asyncio.create_task(self.start(try_n + 1)) return if whoami.user_id != self.id: self.log.error(f"User ID mismatch: expected {self.id}, but got {whoami.user_id}") @@ -350,38 +349,11 @@ class Client(DBClient): } async def _handle_tombstone(self, evt: StateEvent) -> None: - if evt.state_key != "": - return if not evt.content.replacement_room: self.log.info(f"{evt.room_id} tombstoned with no replacement, ignoring") return - is_joined = await self.client.state_store.is_joined( - evt.content.replacement_room, - self.client.mxid, - ) - if is_joined: - self.log.debug( - f"Ignoring tombstone from {evt.room_id} to {evt.content.replacement_room} " - f"sent by {evt.sender}: already joined to replacement room" - ) - return - self.log.debug( - f"Following tombstone from {evt.room_id} to {evt.content.replacement_room} " - f"sent by {evt.sender}" - ) _, server = self.client.parse_user_id(evt.sender) - room_id = await self.client.join_room(evt.content.replacement_room, servers=[server]) - power_levels = await self.client.get_state_event(room_id, EventType.ROOM_POWER_LEVELS) - if power_levels.get_user_level(evt.sender) < power_levels.invite: - self.log.warning( - f"{evt.room_id} was tombstoned into {room_id} by {evt.sender}," - " but the sender doesn't have invite power levels, leaving..." - ) - await self.client.leave_room( - room_id, - f"Followed tombstone from {evt.room_id} by {evt.sender}," - " but sender doesn't have sufficient power level for invites", - ) + await self.client.join_room(evt.content.replacement_room, servers=[server]) async def _handle_invite(self, evt: StrippedStateEvent) -> None: if evt.state_key == self.id and evt.content.membership == Membership.INVITE: diff --git a/maubot/config.py b/maubot/config.py index b8e42de..e11fe1c 100644 --- a/maubot/config.py +++ b/maubot/config.py @@ -32,11 +32,7 @@ class Config(BaseFileConfig): def do_update(self, helper: ConfigUpdateHelper) -> None: base = helper.base copy = helper.copy - - if "database" in self and self["database"].startswith("sqlite:///"): - helper.base["database"] = self["database"].replace("sqlite:///", "sqlite:") - else: - copy("database") + copy("database") copy("database_opts") if isinstance(self["crypto_database"], dict): if self["crypto_database.type"] == "postgres": @@ -56,9 +52,11 @@ class Config(BaseFileConfig): copy("server.port") copy("server.public_url") copy("server.listen") + copy("server.base_path") copy("server.ui_base_path") copy("server.plugin_base_path") copy("server.override_resource_path") + copy("server.appservice_base_path") shared_secret = self["server.unshared_secret"] if shared_secret is None or shared_secret == "generate": base["server.unshared_secret"] = self._new_token() diff --git a/maubot/db/__init__.py b/maubot/db/__init__.py index 68833ce..d6aeb09 100644 --- a/maubot/db/__init__.py +++ b/maubot/db/__init__.py @@ -1,7 +1,7 @@ from mautrix.util.async_db import Database from .client import Client -from .instance import DatabaseEngine, Instance +from .instance import Instance from .upgrade import upgrade_table @@ -10,4 +10,4 @@ def init(db: Database) -> None: table.db = db -__all__ = ["upgrade_table", "init", "Client", "Instance", "DatabaseEngine"] +__all__ = ["upgrade_table", "init", "Client", "Instance"] diff --git a/maubot/db/instance.py b/maubot/db/instance.py index 5bb3f6a..dff7064 100644 --- a/maubot/db/instance.py +++ b/maubot/db/instance.py @@ -16,7 +16,6 @@ from __future__ import annotations from typing import TYPE_CHECKING, ClassVar -from enum import Enum from asyncpg import Record from attr import dataclass @@ -27,11 +26,6 @@ from mautrix.util.async_db import Database fake_db = Database.create("") if TYPE_CHECKING else None -class DatabaseEngine(Enum): - SQLITE = "sqlite" - POSTGRES = "postgres" - - @dataclass class Instance: db: ClassVar[Database] = fake_db @@ -41,31 +35,21 @@ class Instance: enabled: bool primary_user: UserID config_str: str - database_engine: DatabaseEngine | None - - @property - def database_engine_str(self) -> str | None: - return self.database_engine.value if self.database_engine else None @classmethod def _from_row(cls, row: Record | None) -> Instance | None: if row is None: return None - data = {**row} - db_engine = data.pop("database_engine", None) - return cls(**data, database_engine=DatabaseEngine(db_engine) if db_engine else None) - - _columns = "id, type, enabled, primary_user, config, database_engine" + return cls(**row) @classmethod async def all(cls) -> list[Instance]: - q = f"SELECT {cls._columns} FROM instance" - rows = await cls.db.fetch(q) + rows = await cls.db.fetch("SELECT id, type, enabled, primary_user, config FROM instance") return [cls._from_row(row) for row in rows] @classmethod async def get(cls, id: str) -> Instance | None: - q = f"SELECT {cls._columns} FROM instance WHERE id=$1" + q = "SELECT id, type, enabled, primary_user, config FROM instance WHERE id=$1" return cls._from_row(await cls.db.fetchrow(q, id)) async def update_id(self, new_id: str) -> None: @@ -74,27 +58,17 @@ class Instance: @property def _values(self): - return ( - self.id, - self.type, - self.enabled, - self.primary_user, - self.config_str, - self.database_engine_str, - ) + return self.id, self.type, self.enabled, self.primary_user, self.config_str async def insert(self) -> None: q = ( - "INSERT INTO instance (id, type, enabled, primary_user, config, database_engine) " - "VALUES ($1, $2, $3, $4, $5, $6)" + "INSERT INTO instance (id, type, enabled, primary_user, config) " + "VALUES ($1, $2, $3, $4, $5)" ) await self.db.execute(q, *self._values) async def update(self) -> None: - q = """ - UPDATE instance SET type=$2, enabled=$3, primary_user=$4, config=$5, database_engine=$6 - WHERE id=$1 - """ + q = "UPDATE instance SET type=$2, enabled=$3, primary_user=$4, config=$5 WHERE id=$1" await self.db.execute(q, *self._values) async def delete(self) -> None: diff --git a/maubot/db/upgrade/__init__.py b/maubot/db/upgrade/__init__.py index ed96422..146e713 100644 --- a/maubot/db/upgrade/__init__.py +++ b/maubot/db/upgrade/__init__.py @@ -2,4 +2,4 @@ from mautrix.util.async_db import UpgradeTable upgrade_table = UpgradeTable() -from . import v01_initial_revision, v02_instance_database_engine +from . import v01_initial_revision diff --git a/maubot/db/upgrade/v02_instance_database_engine.py b/maubot/db/upgrade/v02_instance_database_engine.py deleted file mode 100644 index 7d2d7e7..0000000 --- a/maubot/db/upgrade/v02_instance_database_engine.py +++ /dev/null @@ -1,25 +0,0 @@ -# maubot - A plugin-based Matrix bot system. -# Copyright (C) 2022 Tulir Asokan -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -from __future__ import annotations - -from mautrix.util.async_db import Connection - -from . import upgrade_table - - -@upgrade_table.register(description="Store instance database engine") -async def upgrade_v2(conn: Connection) -> None: - await conn.execute("ALTER TABLE instance ADD COLUMN database_engine TEXT") diff --git a/maubot/example-config.yaml b/maubot/example-config.yaml index 0a6c8ac..d157269 100644 --- a/maubot/example-config.yaml +++ b/maubot/example-config.yaml @@ -1,8 +1,9 @@ # The full URI to the database. SQLite and Postgres are fully supported. +# Other DBMSes supported by SQLAlchemy may or may not work. # Format examples: -# SQLite: sqlite:filename.db +# SQLite: sqlite:///filename.db # Postgres: postgresql://username:password@hostname/dbname -database: sqlite:maubot.db +database: sqlite:///maubot.db # Separate database URL for the crypto database. "default" means use the same database as above. crypto_database: default @@ -54,6 +55,8 @@ server: port: 29316 # Public base URL where the server is visible. public_url: https://example.com + # The base management API path. + base_path: /_matrix/maubot/v1 # The base path for the UI. ui_base_path: /_matrix/maubot # The base path for plugin endpoints. The instance ID will be appended directly. @@ -61,6 +64,8 @@ server: # Override path from where to load UI resources. # Set to false to using pkg_resources to find the path. override_resource_path: false + # The base appservice API path. Use / for legacy appservice API and /_matrix/app/v1 for v1. + appservice_base_path: /_matrix/app/v1 # The shared secret to sign API access tokens. # Set to "generate" to generate and save a new token at startup. unshared_secret: generate @@ -78,9 +83,8 @@ homeservers: # When this is empty, `mbc auth --register` won't work, but `mbc auth` (login) will. secret: null -# List of administrator users. Each key is a username and the value is the password. -# Plaintext passwords will be bcrypted on startup. Set empty password to prevent normal login. -# Root is a special user that can't have a password and will always exist. +# List of administrator users. Plaintext passwords will be bcrypted on startup. Set empty password +# to prevent normal login. Root is a special user that can't have a password and will always exist. admins: root: "" diff --git a/maubot/handlers/command.py b/maubot/handlers/command.py index 27e6547..495bc27 100644 --- a/maubot/handlers/command.py +++ b/maubot/handlers/command.py @@ -72,7 +72,7 @@ class CommandHandler: self.__mb_must_consume_args__: bool = True self.__mb_arg_fallthrough__: bool = True self.__mb_event_handler__: bool = True - self.__mb_event_types__: set[EventType] = {EventType.ROOM_MESSAGE} + self.__mb_event_type__: EventType = EventType.ROOM_MESSAGE self.__mb_msgtypes__: Iterable[MessageType] = (MessageType.TEXT,) self.__bound_copies__: Dict[Any, CommandHandler] = {} self.__bound_instance__: Any = None @@ -92,10 +92,9 @@ class CommandHandler: "get_name", "is_command_match", "require_subcommand", - "must_consume_args", "arg_fallthrough", "event_handler", - "event_types", + "event_type", "msgtypes", ] for key in keys: @@ -315,7 +314,7 @@ def new( func.__mb_require_subcommand__ = require_subcommand func.__mb_arg_fallthrough__ = arg_fallthrough func.__mb_must_consume_args__ = must_consume_args - func.__mb_event_types__ = {event_type} + func.__mb_event_type__ = event_type if msgtypes: func.__mb_msgtypes__ = msgtypes return func diff --git a/maubot/handlers/event.py b/maubot/handlers/event.py index 9be89b1..a9f8ac8 100644 --- a/maubot/handlers/event.py +++ b/maubot/handlers/event.py @@ -27,12 +27,9 @@ def on(var: EventType | InternalEventType | EventHandler) -> EventHandlerDecorat def decorator(func: EventHandler) -> EventHandler: func.__mb_event_handler__ = True if isinstance(var, (EventType, InternalEventType)): - if hasattr(func, "__mb_event_types__"): - func.__mb_event_types__.add(var) - else: - func.__mb_event_types__ = {var} + func.__mb_event_type__ = var else: - func.__mb_event_types__ = {EventType.ALL} + func.__mb_event_type__ = EventType.ALL return func diff --git a/maubot/instance.py b/maubot/instance.py index 8427e3c..b9b1c23 100644 --- a/maubot/instance.py +++ b/maubot/instance.py @@ -25,17 +25,16 @@ import os.path from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap +import sqlalchemy as sql from mautrix.types import UserID -from mautrix.util import background_task from mautrix.util.async_db import Database, Scheme, UpgradeTable from mautrix.util.async_getter_lock import async_getter_lock from mautrix.util.config import BaseProxyConfig, RecursiveDict from mautrix.util.logging import TraceLogger from .client import Client -from .db import DatabaseEngine, Instance as DBInstance -from .lib.optionalalchemy import Engine, MetaData, create_engine +from .db import Instance as DBInstance from .lib.plugin_db import ProxyPostgresDatabase from .loader import DatabaseType, PluginLoader, ZippedPluginLoader from .plugin_base import Plugin @@ -72,21 +71,10 @@ class PluginInstance(DBInstance): started: bool def __init__( - self, - id: str, - type: str, - enabled: bool, - primary_user: UserID, - config: str = "", - database_engine: DatabaseEngine | None = None, + self, id: str, type: str, enabled: bool, primary_user: UserID, config: str = "" ) -> None: super().__init__( - id=id, - type=type, - enabled=bool(enabled), - primary_user=primary_user, - config_str=config, - database_engine=database_engine, + id=id, type=type, enabled=bool(enabled), primary_user=primary_user, config_str=config ) def __hash__(self) -> int: @@ -123,12 +111,10 @@ class PluginInstance(DBInstance): "database": ( self.inst_db is not None and self.maubot.config["api_features.instance_database"] ), - "database_interface": self.loader.meta.database_type_str if self.loader else "unknown", - "database_engine": self.database_engine_str, } def _introspect_sqlalchemy(self) -> dict: - metadata = MetaData() + metadata = sql.MetaData() metadata.reflect(self.inst_db) return { table.name: { @@ -214,7 +200,7 @@ class PluginInstance(DBInstance): async def get_db_tables(self) -> dict: if self.inst_db_tables is None: - if isinstance(self.inst_db, Engine): + if isinstance(self.inst_db, sql.engine.Engine): self.inst_db_tables = self._introspect_sqlalchemy() elif self.inst_db.scheme == Scheme.SQLITE: self.inst_db_tables = await self._introspect_sqlite() @@ -277,37 +263,18 @@ class PluginInstance(DBInstance): def save_config(self, data: RecursiveDict[CommentedMap]) -> None: buf = io.StringIO() yaml.dump(data, buf) - val = buf.getvalue() - if val != self.config_str: - self.config_str = val - self.log.debug("Creating background task to save updated config") - background_task.create(self.update()) + self.config_str = buf.getvalue() async def start_database( self, upgrade_table: UpgradeTable | None = None, actually_start: bool = True ) -> None: if self.loader.meta.database_type == DatabaseType.SQLALCHEMY: - if self.database_engine is None: - await self.update_db_engine(DatabaseEngine.SQLITE) - elif self.database_engine == DatabaseEngine.POSTGRES: - raise RuntimeError( - "Instance database engine is marked as Postgres, but plugin uses legacy " - "database interface, which doesn't support postgres." - ) - self.inst_db = create_engine(f"sqlite:///{self._sqlite_db_path}") + self.inst_db = sql.create_engine(f"sqlite:///{self._sqlite_db_path}") elif self.loader.meta.database_type == DatabaseType.ASYNCPG: - if self.database_engine is None: - if os.path.exists(self._sqlite_db_path) or not self.maubot.plugin_postgres_db: - await self.update_db_engine(DatabaseEngine.SQLITE) - else: - await self.update_db_engine(DatabaseEngine.POSTGRES) instance_db_log = db_log.getChild(self.id) - if self.database_engine == DatabaseEngine.POSTGRES: - if not self.maubot.plugin_postgres_db: - raise RuntimeError( - "Instance database engine is marked as Postgres, but this maubot isn't " - "configured to support Postgres for plugin databases" - ) + # TODO should there be a way to choose between SQLite and Postgres + # for individual instances? Maybe checking the existence of the SQLite file. + if self.maubot.plugin_postgres_db: self.inst_db = ProxyPostgresDatabase( pool=self.maubot.plugin_postgres_db, instance_id=self.id, @@ -317,7 +284,7 @@ class PluginInstance(DBInstance): ) else: self.inst_db = Database.create( - f"sqlite:{self._sqlite_db_path}", + f"sqlite:///{self._sqlite_db_path}", upgrade_table=upgrade_table, log=instance_db_log, ) @@ -329,7 +296,7 @@ class PluginInstance(DBInstance): async def stop_database(self) -> None: if isinstance(self.inst_db, Database): await self.inst_db.stop() - elif isinstance(self.inst_db, Engine): + elif isinstance(self.inst_db, sql.engine.Engine): self.inst_db.dispose() else: raise RuntimeError(f"Unknown database type {type(self.inst_db).__name__}") @@ -367,12 +334,7 @@ class PluginInstance(DBInstance): self.log.debug("Disabling webapp after plugin meta reload") self.disable_webapp() if self.loader.meta.database: - try: - await self.start_database(cls.get_db_upgrade_table()) - except Exception: - self.log.exception("Failed to start instance database") - await self.update_enabled(False) - return + await self.start_database(cls.get_db_upgrade_table()) config_class = cls.get_config_class() if config_class: try: @@ -493,11 +455,6 @@ class PluginInstance(DBInstance): self.enabled = enabled await self.update() - async def update_db_engine(self, db_engine: DatabaseEngine | None) -> None: - if db_engine is not None and db_engine != self.database_engine: - self.database_engine = db_engine - await self.update() - @classmethod @async_getter_lock async def get( diff --git a/maubot/lib/optionalalchemy.py b/maubot/lib/optionalalchemy.py deleted file mode 100644 index ba94271..0000000 --- a/maubot/lib/optionalalchemy.py +++ /dev/null @@ -1,19 +0,0 @@ -try: - from sqlalchemy import MetaData, asc, create_engine, desc - from sqlalchemy.engine import Engine - from sqlalchemy.exc import IntegrityError, OperationalError -except ImportError: - - class FakeError(Exception): - pass - - class FakeType: - def __init__(self, *args, **kwargs): - raise Exception("SQLAlchemy is not installed") - - def create_engine(*args, **kwargs): - raise Exception("SQLAlchemy is not installed") - - MetaData = Engine = FakeType - IntegrityError = OperationalError = FakeError - asc = desc = lambda a: a diff --git a/maubot/lib/zipimport.py b/maubot/lib/zipimport.py index e7b77db..963f14f 100644 --- a/maubot/lib/zipimport.py +++ b/maubot/lib/zipimport.py @@ -323,7 +323,6 @@ _zip_searchorder = ( (".py", False, False), ) - # Given a module name, return the potential file path in the # archive (without extension). def _get_module_path(self, fullname): @@ -352,7 +351,6 @@ def _get_module_info(self, fullname): # implementation - # _read_directory(archive) -> files dict (new reference) # # Given a path to a Zip archive, build a dict, mapping file names @@ -526,7 +524,6 @@ cp437_table = ( _importing_zlib = False - # Return the zlib.decompress function object, or NULL if zlib couldn't # be imported. The function is cached when found, so subsequent calls # don't import zlib again. diff --git a/maubot/loader/meta.py b/maubot/loader/meta.py index d368e24..f16937b 100644 --- a/maubot/loader/meta.py +++ b/maubot/loader/meta.py @@ -13,7 +13,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import List, Optional +from typing import List from attr import dataclass from packaging.version import InvalidVersion, Version @@ -63,7 +63,3 @@ class PluginMeta(SerializableAttrs): extra_files: List[str] = [] dependencies: List[str] = [] soft_dependencies: List[str] = [] - - @property - def database_type_str(self) -> Optional[str]: - return self.database_type.value if self.database else None diff --git a/maubot/loader/zip.py b/maubot/loader/zip.py index 8642183..70cee5a 100644 --- a/maubot/loader/zip.py +++ b/maubot/loader/zip.py @@ -31,7 +31,7 @@ from ..config import Config from ..lib.zipimport import ZipImportError, zipimporter from ..plugin_base import Plugin from .abc import IDConflictError, PluginClass, PluginLoader -from .meta import DatabaseType, PluginMeta +from .meta import PluginMeta current_version = Version(__version__) yaml = YAML() @@ -155,9 +155,9 @@ class ZippedPluginLoader(PluginLoader): return file, meta @classmethod - def verify_meta(cls, source) -> tuple[str, Version, DatabaseType | None]: + def verify_meta(cls, source) -> tuple[str, Version]: _, meta = cls._read_meta(source) - return meta.id, meta.version, meta.database_type if meta.database else None + return meta.id, meta.version def _load_meta(self) -> None: file, meta = self._read_meta(self.path) @@ -167,7 +167,7 @@ class ZippedPluginLoader(PluginLoader): if "/" in meta.main_class: self.main_module, self.main_class = meta.main_class.split("/")[:2] else: - self.main_module = meta.modules[-1] + self.main_module = meta.modules[0] self.main_class = meta.main_class self._file = file diff --git a/maubot/management/api/client.py b/maubot/management/api/client.py index d2ad35d..d95286b 100644 --- a/maubot/management/api/client.py +++ b/maubot/management/api/client.py @@ -16,7 +16,6 @@ from __future__ import annotations from json import JSONDecodeError -import logging from aiohttp import web @@ -28,8 +27,6 @@ from ...client import Client from .base import routes from .responses import resp -log = logging.getLogger("maubot.server.client") - @routes.get("/clients") async def get_clients(_: web.Request) -> web.Response: @@ -57,13 +54,11 @@ async def _create_client(user_id: UserID | None, data: dict) -> web.Response: ) try: whoami = await new_client.whoami() - except MatrixInvalidToken as e: + except MatrixInvalidToken: return resp.bad_client_access_token except MatrixRequestError: - log.warning(f"Failed to get whoami from {homeserver} for new client", exc_info=True) return resp.bad_client_access_details except MatrixConnectionError: - log.warning(f"Failed to connect to {homeserver} for new client", exc_info=True) return resp.bad_client_connection_details if user_id is None: existing_client = await Client.get(whoami.user_id) @@ -78,8 +73,8 @@ async def _create_client(user_id: UserID | None, data: dict) -> web.Response: ) client.enabled = data.get("enabled", True) client.sync = data.get("sync", True) - await client.update_autojoin(data.get("autojoin", True), save=False) - await client.update_online(data.get("online", True), save=False) + client.autojoin = data.get("autojoin", True) + client.online = data.get("online", True) client.displayname = data.get("displayname", "disable") client.avatar_url = data.get("avatar_url", "disable") await client.update() @@ -95,12 +90,8 @@ async def _update_client(client: Client, data: dict, is_login: bool = False) -> except MatrixInvalidToken: return resp.bad_client_access_token except MatrixRequestError: - log.warning( - f"Failed to get whoami from homeserver to update client details", exc_info=True - ) return resp.bad_client_access_details except MatrixConnectionError: - log.warning(f"Failed to connect to homeserver to update client details", exc_info=True) return resp.bad_client_connection_details except ValueError as e: str_err = str(e) diff --git a/maubot/management/api/client_auth.py b/maubot/management/api/client_auth.py index 4e5e201..c5baade 100644 --- a/maubot/management/api/client_auth.py +++ b/maubot/management/api/client_auth.py @@ -184,10 +184,11 @@ async def _do_sso(req: AuthRequestInfo) -> web.Response: cfg = get_config() public_url = ( URL(cfg["server.public_url"]) - / "_matrix/maubot/v1/client/auth_external_sso/complete" + / cfg["server.base_path"].lstrip("/") + / "client/auth_external_sso/complete" / waiter_id ) - sso_url = req.client.api.base_url.with_path(str(Path.v3.login.sso.redirect)).with_query( + sso_url = req.client.api.base_url.with_path(str(Path.login.sso.redirect)).with_query( {"redirectUrl": str(public_url)} ) sso_waiters[waiter_id] = req, asyncio.get_running_loop().create_future() diff --git a/maubot/management/api/instance_database.py b/maubot/management/api/instance_database.py index 2f8c37a..97b2edf 100644 --- a/maubot/management/api/instance_database.py +++ b/maubot/management/api/instance_database.py @@ -19,12 +19,12 @@ from datetime import datetime from aiohttp import web from asyncpg import PostgresError +from sqlalchemy import asc, desc, engine, exc import aiosqlite from mautrix.util.async_db import Database from ...instance import PluginInstance -from ...lib.optionalalchemy import Engine, IntegrityError, OperationalError, asc, desc from .base import routes from .responses import resp @@ -56,17 +56,15 @@ async def get_table(request: web.Request) -> web.Response: try: order = [tuple(order.split(":")) for order in request.query.getall("order")] order = [ - ( - (asc if sort.lower() == "asc" else desc)(table.columns[column]) - if sort - else table.columns[column] - ) + (asc if sort.lower() == "asc" else desc)(table.columns[column]) + if sort + else table.columns[column] for column, sort in order ] except KeyError: order = [] limit = int(request.query.get("limit", "100")) - if isinstance(instance.inst_db, Engine): + if isinstance(instance.inst_db, engine.Engine): return _execute_query_sqlalchemy(instance, table.select().order_by(*order).limit(limit)) @@ -84,7 +82,7 @@ async def query(request: web.Request) -> web.Response: except KeyError: return resp.query_missing rows_as_dict = data.get("rows_as_dict", False) - if isinstance(instance.inst_db, Engine): + if isinstance(instance.inst_db, engine.Engine): return _execute_query_sqlalchemy(instance, sql_query, rows_as_dict) elif isinstance(instance.inst_db, Database): try: @@ -133,12 +131,12 @@ async def _execute_query_asyncpg( def _execute_query_sqlalchemy( instance: PluginInstance, sql_query: str, rows_as_dict: bool = False ) -> web.Response: - assert isinstance(instance.inst_db, Engine) + assert isinstance(instance.inst_db, engine.Engine) try: res = instance.inst_db.execute(sql_query) - except IntegrityError as e: + except exc.IntegrityError as e: return resp.sql_integrity_error(e, sql_query) - except OperationalError as e: + except exc.OperationalError as e: return resp.sql_operational_error(e, sql_query) data = { "ok": True, diff --git a/maubot/management/api/log.py b/maubot/management/api/log.py index 14c80cd..1f0b4bb 100644 --- a/maubot/management/api/log.py +++ b/maubot/management/api/log.py @@ -22,8 +22,6 @@ import logging from aiohttp import web, web_ws -from mautrix.util import background_task - from .auth import is_valid_token from .base import routes @@ -144,7 +142,7 @@ async def log_websocket(request: web.Request) -> web.WebSocketResponse: await ws.close(code=4000) log.debug(f"Connection from {request.remote} terminated due to no authentication") - background_task.create(close_if_not_authenticated()) + asyncio.create_task(close_if_not_authenticated()) try: msg: web_ws.WSMessage diff --git a/maubot/management/api/middleware.py b/maubot/management/api/middleware.py index 17141fa..0ecb681 100644 --- a/maubot/management/api/middleware.py +++ b/maubot/management/api/middleware.py @@ -29,7 +29,7 @@ log = logging.getLogger("maubot.server") @web.middleware async def auth(request: web.Request, handler: Handler) -> web.Response: - subpath = request.path[len("/_matrix/maubot/v1") :] + subpath = request.path[len(get_config()["server.base_path"]) :] if ( subpath.startswith("/auth/") or subpath.startswith("/client/auth_external_sso/complete/") diff --git a/maubot/management/api/plugin_upload.py b/maubot/management/api/plugin_upload.py index 4cd2c47..ea4fd1f 100644 --- a/maubot/management/api/plugin_upload.py +++ b/maubot/management/api/plugin_upload.py @@ -23,17 +23,10 @@ import traceback from aiohttp import web from packaging.version import Version -from ...loader import DatabaseType, MaubotZipImportError, PluginLoader, ZippedPluginLoader +from ...loader import MaubotZipImportError, PluginLoader, ZippedPluginLoader from .base import get_config, routes from .responses import resp -try: - import sqlalchemy - - has_alchemy = True -except ImportError: - has_alchemy = False - log = logging.getLogger("maubot.server.upload") @@ -43,11 +36,9 @@ async def put_plugin(request: web.Request) -> web.Response: content = await request.read() file = BytesIO(content) try: - pid, version, db_type = ZippedPluginLoader.verify_meta(file) + pid, version = ZippedPluginLoader.verify_meta(file) except MaubotZipImportError as e: return resp.plugin_import_error(str(e), traceback.format_exc()) - if db_type == DatabaseType.SQLALCHEMY and not has_alchemy: - return resp.sqlalchemy_not_installed if pid != plugin_id: return resp.pid_mismatch plugin = PluginLoader.id_cache.get(plugin_id, None) @@ -64,11 +55,9 @@ async def upload_plugin(request: web.Request) -> web.Response: content = await request.read() file = BytesIO(content) try: - pid, version, db_type = ZippedPluginLoader.verify_meta(file) + pid, version = ZippedPluginLoader.verify_meta(file) except MaubotZipImportError as e: return resp.plugin_import_error(str(e), traceback.format_exc()) - if db_type == DatabaseType.SQLALCHEMY and not has_alchemy: - return resp.sqlalchemy_not_installed plugin = PluginLoader.id_cache.get(pid, None) if not plugin: return await upload_new_plugin(content, pid, version) diff --git a/maubot/management/api/responses.py b/maubot/management/api/responses.py index 0f22caa..15f6a96 100644 --- a/maubot/management/api/responses.py +++ b/maubot/management/api/responses.py @@ -15,16 +15,13 @@ # along with this program. If not, see . from __future__ import annotations -from typing import TYPE_CHECKING from http import HTTPStatus from aiohttp import web from asyncpg import PostgresError +from sqlalchemy.exc import IntegrityError, OperationalError import aiosqlite -if TYPE_CHECKING: - from sqlalchemy.exc import IntegrityError, OperationalError - class _Response: @property @@ -327,16 +324,6 @@ class _Response: } ) - @property - def sqlalchemy_not_installed(self) -> web.Response: - return web.json_response( - { - "error": "This plugin requires a legacy database, but SQLAlchemy is not installed", - "errcode": "unsupported_plugin_database", - }, - status=HTTPStatus.NOT_IMPLEMENTED, - ) - @property def table_not_found(self) -> web.Response: return web.json_response( diff --git a/maubot/management/frontend/src/api.js b/maubot/management/frontend/src/api.js index d1a51b8..5c1fd55 100644 --- a/maubot/management/frontend/src/api.js +++ b/maubot/management/frontend/src/api.js @@ -205,7 +205,7 @@ export const getClients = () => defaultGet("/clients") export const getClient = id => defaultGet(`/clients/${id}`) export async function uploadAvatar(id, data, mime) { - const resp = await fetch(`${BASE_PATH}/proxy/${id}/_matrix/media/v3/upload`, { + const resp = await fetch(`${BASE_PATH}/proxy/${id}/_matrix/media/r0/upload`, { headers: getHeaders(mime), body: data, method: "POST", @@ -217,9 +217,8 @@ export function getAvatarURL({ id, avatar_url }) { if (!avatar_url?.startsWith("mxc://")) { return null } - avatar_url = avatar_url.substring("mxc://".length) - // Note: the maubot backend will replace the query param with an authorization header - return `${BASE_PATH}/proxy/${id}/_matrix/client/v1/media/download/${avatar_url}?access_token=${ + avatar_url = avatar_url.substr("mxc://".length) + return `${BASE_PATH}/proxy/${id}/_matrix/media/r0/download/${avatar_url}?access_token=${ localStorage.accessToken}` } diff --git a/maubot/management/frontend/src/pages/Main.js b/maubot/management/frontend/src/pages/Main.js index 5ee374e..fb650b6 100644 --- a/maubot/management/frontend/src/pages/Main.js +++ b/maubot/management/frontend/src/pages/Main.js @@ -45,8 +45,9 @@ class Main extends Component { const resp = await fetch(process.env.PUBLIC_URL + "/paths.json", { headers: { "Content-Type": "application/json" }, }) - const apiPaths = await resp.json() - api.setBasePath(apiPaths.api_path) + const apiPathJson = await resp.json() + const apiPath = apiPathJson.api_path + api.setBasePath(`${apiPath}`) } catch (err) { console.error("Failed to get API path:", err) } diff --git a/maubot/management/frontend/src/pages/dashboard/Instance.js b/maubot/management/frontend/src/pages/dashboard/Instance.js index 049c5e5..c487fb0 100644 --- a/maubot/management/frontend/src/pages/dashboard/Instance.js +++ b/maubot/management/frontend/src/pages/dashboard/Instance.js @@ -43,7 +43,7 @@ class Instance extends BaseMainView { } get entryKeys() { - return ["id", "primary_user", "enabled", "started", "type", "config", "database_engine"] + return ["id", "primary_user", "enabled", "started", "type", "config"] } get initialState() { @@ -54,7 +54,6 @@ class Instance extends BaseMainView { started: true, type: "", config: "", - database_engine: "", saving: false, deleting: false, diff --git a/maubot/management/frontend/src/pages/dashboard/Log.js b/maubot/management/frontend/src/pages/dashboard/Log.js index b80e6ad..5cd9833 100644 --- a/maubot/management/frontend/src/pages/dashboard/Log.js +++ b/maubot/management/frontend/src/pages/dashboard/Log.js @@ -41,7 +41,7 @@ class LogEntry extends PureComponent { const req = this.props.line.matrix_http_request return <> - {req.method} {req.url || req.path} + {req.method} {req.path}
{Object.entries(req.content || {}).length > 0 && } diff --git a/maubot/matrix.py b/maubot/matrix.py index a99c1a4..9c90c8f 100644 --- a/maubot/matrix.py +++ b/maubot/matrix.py @@ -24,7 +24,6 @@ import attr from mautrix.client import Client as MatrixClient, SyncStream from mautrix.errors import DecryptionError from mautrix.types import ( - BaseMessageEventContentFuncs, EncryptedEvent, Event, EventID, @@ -62,10 +61,7 @@ async def parse_formatted( html = message else: return message, escape(message) - text = (await MaubotHTMLParser().parse(html)).text - if len(text) + len(html) > 40000: - text = text[:100] + "[long message cut off]" - return text, html + return (await MaubotHTMLParser().parse(html)).text, html class MaubotMessageEvent(MessageEvent): @@ -86,31 +82,8 @@ class MaubotMessageEvent(MessageEvent): markdown: bool = True, allow_html: bool = False, reply: bool | str = False, - in_thread: bool | None = None, edits: EventID | MessageEvent | None = None, ) -> EventID: - """ - Respond to the message. - - Args: - content: The content to respond with. If this is a string, it will be passed to - :func:`parse_formatted` with the markdown and allow_html flags. - Otherwise, the content is used as-is - event_type: The type of event to send. - markdown: When content is a string, should it be parsed as markdown? - allow_html: When content is a string, should it allow raw HTML? - reply: Should the response be sent as a reply to this event? - in_thread: Should the response be sent in a thread with this event? - By default (``None``), the response will be in a thread if this event is in a - thread. If set to ``False``, the response will never be in a thread. If set to - ``True``, the response will always be in a thread, creating one with this event as - the root if necessary. - edits: An event ID or MessageEvent to edit. If set, the reply and in_thread parameters - are ignored, as edits can't change the reply or thread status. - - Returns: - The ID of the response event. - """ if isinstance(content, str): content = TextMessageEventContent(msgtype=MessageType.NOTICE, body=content) if allow_html or markdown: @@ -120,19 +93,7 @@ class MaubotMessageEvent(MessageEvent): ) if edits: content.set_edit(edits) - if ( - not edits - and in_thread is not False - and ( - in_thread - or ( - isinstance(self.content, BaseMessageEventContentFuncs) - and self.content.get_thread_parent() - ) - ) - ): - content.set_thread_parent(self) - if reply and not edits: + elif reply: if reply != "force" and self.disable_reply: content.body = f"{self.sender}: {content.body}" fmt_body = content.formatted_body or escape(content.body).replace("\n", "
") @@ -151,71 +112,17 @@ class MaubotMessageEvent(MessageEvent): event_type: EventType = EventType.ROOM_MESSAGE, markdown: bool = True, allow_html: bool = False, - in_thread: bool | None = None, ) -> Awaitable[EventID]: - """ - Reply to the message. The parameters are the same as :meth:`respond`, - but ``reply`` is always ``True`` and ``edits`` is not supported. - - Args: - content: The content to respond with. If this is a string, it will be passed to - :func:`parse_formatted` with the markdown and allow_html flags. - Otherwise, the content is used as-is - event_type: The type of event to send. - markdown: When content is a string, should it be parsed as markdown? - allow_html: When content is a string, should it allow raw HTML? - in_thread: Should the response be sent in a thread with this event? - By default (``None``), the response will be in a thread if this event is in a - thread. If set to ``False``, the response will never be in a thread. If set to - ``True``, the response will always be in a thread, creating one with this event as - the root if necessary. - - Returns: - The ID of the response event. - """ return self.respond( - content, - event_type, - markdown=markdown, - reply=True, - in_thread=in_thread, - allow_html=allow_html, + content, event_type, markdown=markdown, reply=True, allow_html=allow_html ) def mark_read(self) -> Awaitable[None]: - """ - Mark this event as read. - """ return self.client.send_receipt(self.room_id, self.event_id, "m.read") def react(self, key: str) -> Awaitable[EventID]: - """ - React to this event with the given key. - - Args: - key: The key to react with. Often an unicode emoji. - - Returns: - The ID of the reaction event. - - Examples: - >>> evt: MaubotMessageEvent - >>> evt.react("🐈️") - """ return self.client.react(self.room_id, self.event_id, key) - def redact(self, reason: str | None = None) -> Awaitable[EventID]: - """ - Redact this event. - - Args: - reason: Optionally, the reason for redacting the event. - - Returns: - The ID of the redaction event. - """ - return self.client.redact(self.room_id, self.event_id, reason=reason) - def edit( self, content: str | MessageEventContent, @@ -223,21 +130,6 @@ class MaubotMessageEvent(MessageEvent): markdown: bool = True, allow_html: bool = False, ) -> Awaitable[EventID]: - """ - Edit this event. Note that other clients will only render the edit if it was sent by the - same user who's doing the editing. - - Args: - content: The new content for the event. If this is a string, it will be passed to - :func:`parse_formatted` with the markdown and allow_html flags. - Otherwise, the content is used as-is. - event_type: The type of event to edit into. - markdown: When content is a string, should it be parsed as markdown? - allow_html: When content is a string, should it allow raw HTML? - - Returns: - The ID of the edit event. - """ return self.respond( content, event_type, markdown=markdown, edits=self, allow_html=allow_html ) diff --git a/maubot/plugin_base.py b/maubot/plugin_base.py index 1be15e0..396a7b6 100644 --- a/maubot/plugin_base.py +++ b/maubot/plugin_base.py @@ -20,17 +20,14 @@ from abc import ABC from asyncio import AbstractEventLoop from aiohttp import ClientSession +from sqlalchemy.engine.base import Engine from yarl import URL from mautrix.util.async_db import Database, UpgradeTable from mautrix.util.config import BaseProxyConfig from mautrix.util.logging import TraceLogger -from .scheduler import BasicScheduler - if TYPE_CHECKING: - from sqlalchemy.engine.base import Engine - from .client import MaubotMatrixClient from .loader import BasePluginLoader from .plugin_server import PluginWebApp @@ -43,7 +40,6 @@ class Plugin(ABC): log: TraceLogger loop: AbstractEventLoop loader: BasePluginLoader - sched: BasicScheduler config: BaseProxyConfig | None database: Engine | Database | None webapp: PluginWebApp | None @@ -57,12 +53,11 @@ class Plugin(ABC): instance_id: str, log: TraceLogger, config: BaseProxyConfig | None, - database: Engine | Database | None, + database: Engine | None, webapp: PluginWebApp | None, webapp_url: str | None, loader: BasePluginLoader, ) -> None: - self.sched = BasicScheduler(log=log.getChild("scheduler")) self.client = client self.loop = loop self.http = http @@ -81,9 +76,8 @@ class Plugin(ABC): val = getattr(obj, key) try: if val.__mb_event_handler__: - for event_type in val.__mb_event_types__: - self._handlers_at_startup.append((val, event_type)) - self.client.add_event_handler(event_type, val) + self._handlers_at_startup.append((val, val.__mb_event_type__)) + self.client.add_event_handler(val.__mb_event_type__, val) except AttributeError: pass try: @@ -122,7 +116,6 @@ class Plugin(ABC): self.client.remove_event_handler(event_type, func) if self.webapp is not None: self.webapp.clear() - self.sched.stop() await self.stop() async def stop(self) -> None: diff --git a/maubot/plugin_server.py b/maubot/plugin_server.py index e5c246c..9dd2df4 100644 --- a/maubot/plugin_server.py +++ b/maubot/plugin_server.py @@ -40,8 +40,6 @@ class PluginWebApp(web.UrlDispatcher): self._resources = [] self._named_resources = {} self._middleware = [] - self._resource_index = {} - self._matched_sub_app_resources = [] async def handle(self, request: web.Request) -> web.StreamResponse: match_info = await self.resolve(request) diff --git a/maubot/py.typed b/maubot/py.typed deleted file mode 100644 index e69de29..0000000 diff --git a/maubot/scheduler.py b/maubot/scheduler.py deleted file mode 100644 index 0cb39ed..0000000 --- a/maubot/scheduler.py +++ /dev/null @@ -1,159 +0,0 @@ -# maubot - A plugin-based Matrix bot system. -# Copyright (C) 2024 Tulir Asokan -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -from __future__ import annotations - -from typing import Awaitable, Callable -import asyncio -import logging - - -class BasicScheduler: - background_loop: asyncio.Task | None - tasks: set[asyncio.Task] - log: logging.Logger - - def __init__(self, log: logging.Logger) -> None: - self.log = log - self.tasks = set() - - def _find_caller(self) -> str: - try: - file_name, line_number, function_name, _ = self.log.findCaller() - return f"{function_name} at {file_name}:{line_number}" - except ValueError: - return "unknown function" - - def run_periodically( - self, - period: float | int, - func: Callable[[], Awaitable], - run_task_in_background: bool = False, - catch_errors: bool = True, - ) -> asyncio.Task: - """ - Run a function periodically in the background. - - Args: - period: The period in seconds between each call to the function. - func: The function to run. No parameters will be provided, - use :meth:`functools.partial` if you need to pass parameters. - run_task_in_background: If ``True``, the function will be run in a background task. - If ``False`` (the default), the loop will wait for the task to return before - sleeping for the next period. - catch_errors: Whether the scheduler should catch and log any errors. - If ``False``, errors will be raised, and the caller must await the returned task - to find errors. This parameter has no effect if ``run_task_in_background`` - is ``True``. - - Returns: - The asyncio task object representing the background loop. - """ - task = asyncio.create_task( - self._call_periodically( - period, - func, - caller=self._find_caller(), - catch_errors=catch_errors, - run_task_in_background=run_task_in_background, - ) - ) - self._register_task(task) - return task - - def run_later( - self, delay: float | int, coro: Awaitable, catch_errors: bool = True - ) -> asyncio.Task: - """ - Run a coroutine after a delay. - - Examples: - >>> self.sched.run_later(5, self.async_task(meow=True)) - - Args: - delay: The delay in seconds to await the coroutine after. - coro: The coroutine to await. - catch_errors: Whether the scheduler should catch and log any errors. - If ``False``, errors will be raised, and the caller must await the returned task - to find errors. - - Returns: - The asyncio task object representing the scheduled task. - """ - task = asyncio.create_task( - self._call_with_delay( - delay, coro, caller=self._find_caller(), catch_errors=catch_errors - ) - ) - self._register_task(task) - return task - - def _register_task(self, task: asyncio.Task) -> None: - self.tasks.add(task) - task.add_done_callback(self.tasks.discard) - - async def _call_periodically( - self, - period: float | int, - func: Callable[[], Awaitable], - caller: str, - catch_errors: bool, - run_task_in_background: bool, - ) -> None: - while True: - try: - await asyncio.sleep(period) - if run_task_in_background: - self._register_task( - asyncio.create_task(self._call_periodically_background(func(), caller)) - ) - else: - await func() - except asyncio.CancelledError: - raise - except Exception: - if catch_errors: - self.log.exception(f"Uncaught error in background loop (created in {caller})") - else: - raise - - async def _call_periodically_background(self, coro: Awaitable, caller: str) -> None: - try: - await coro - except asyncio.CancelledError: - raise - except Exception: - self.log.exception(f"Uncaught error in background loop subtask (created in {caller})") - - async def _call_with_delay( - self, delay: float | int, coro: Awaitable, caller: str, catch_errors: bool - ) -> None: - try: - await asyncio.sleep(delay) - await coro - except asyncio.CancelledError: - raise - except Exception: - if catch_errors: - self.log.exception(f"Uncaught error in scheduled task (created in {caller})") - else: - raise - - def stop(self) -> None: - """ - Stop all scheduled tasks and background loops. - """ - for task in self.tasks: - task.cancel(msg="Scheduler stopped") diff --git a/maubot/server.py b/maubot/server.py index dd1101e..d70ae7e 100644 --- a/maubot/server.py +++ b/maubot/server.py @@ -15,7 +15,6 @@ # along with this program. If not, see . from __future__ import annotations -from typing import Callable import asyncio import json import logging @@ -53,7 +52,7 @@ class MaubotServer: self.config = config self.setup_appservice() - self.app.add_subapp("/_matrix/maubot/v1", management_api) + self.app.add_subapp(config["server.base_path"], management_api) self.setup_instance_subapps() self.setup_management_ui() @@ -64,14 +63,14 @@ class MaubotServer: if request.path.startswith(path): request = request.clone( rel_url=request.rel_url.with_path( - request.rel_url.path[len(path) - 1 :] + request.rel_url.path[len(path) :] ).with_query(request.query_string) ) return await app.handle(request) return web.Response(status=404) def get_instance_subapp(self, instance_id: str) -> tuple[PluginWebApp, str]: - subpath = self.config["server.plugin_base_path"] + instance_id + "/" + subpath = self.config["server.plugin_base_path"] + instance_id url = self.config["server.public_url"] + subpath try: return self.plugin_routes[subpath], url @@ -82,7 +81,7 @@ class MaubotServer: def remove_instance_webapp(self, instance_id: str) -> None: try: - subpath = self.config["server.plugin_base_path"] + instance_id + "/" + subpath = self.config["server.plugin_base_path"] + instance_id self.plugin_routes.pop(subpath).clear() except KeyError: return @@ -94,7 +93,7 @@ class MaubotServer: self.app.router.register_resource(resource) def setup_appservice(self) -> None: - as_path = PathBuilder("/_matrix/appservice/v1") + as_path = PathBuilder(self.config["server.appservice_base_path"]) self.add_route(Method.PUT, as_path.transactions, self.handle_transaction) def setup_management_ui(self) -> None: @@ -128,13 +127,6 @@ class MaubotServer: ) self.app.router.add_get(ui_base, ui_base_redirect) - @staticmethod - def _static_data(data: bytes, mime: str) -> Callable[[web.Request], web.Response]: - def fn(_: web.Request) -> web.Response: - return web.Response(body=data, content_type=mime) - - return fn - def setup_static_root_files(self, directory: str, ui_base: str) -> None: files = { "asset-manifest.json": "application/json", @@ -144,14 +136,20 @@ class MaubotServer: for file, mime in files.items(): with open(f"{directory}/{file}", "rb") as stream: data = stream.read() - self.app.router.add_get(f"{ui_base}/{file}", self._static_data(data, mime)) + self.app.router.add_get( + f"{ui_base}/{file}", lambda _: web.Response(body=data, content_type=mime) + ) + # also set up a resource path for the public url path prefix config + # cut the prefix path from public_url public_url = self.config["server.public_url"] + base_path = self.config["server.base_path"] public_url_path = "" if public_url: public_url_path = URL(public_url).path.rstrip("/") - api_path = f"{public_url_path}/_matrix/maubot/v1" + # assemble with base_path + api_path = f"{public_url_path}{base_path}" path_prefix_response_body = json.dumps({"api_path": api_path.rstrip("/")}) self.app.router.add_get( diff --git a/maubot/standalone/Dockerfile b/maubot/standalone/Dockerfile index 54623f2..8bf06f8 100644 --- a/maubot/standalone/Dockerfile +++ b/maubot/standalone/Dockerfile @@ -1,8 +1,9 @@ -FROM docker.io/alpine:3.21 +FROM docker.io/alpine:3.15 RUN apk add --no-cache \ python3 py3-pip py3-setuptools py3-wheel \ py3-aiohttp \ + py3-sqlalchemy \ py3-attrs \ py3-bcrypt \ py3-cffi \ @@ -25,8 +26,8 @@ RUN cd /opt/maubot \ python3-dev \ libffi-dev \ build-base \ - && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \ + && pip3 install -r requirements.txt -r optional-requirements.txt \ && apk del .build-deps COPY . /opt/maubot -RUN cd /opt/maubot && pip3 install --break-system-packages . +RUN cd /opt/maubot && pip3 install . diff --git a/maubot/standalone/__main__.py b/maubot/standalone/__main__.py index c320af4..40a218d 100644 --- a/maubot/standalone/__main__.py +++ b/maubot/standalone/__main__.py @@ -30,11 +30,7 @@ from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap from yarl import URL -from mautrix.appservice import AppServiceServerMixin -from mautrix.client import SyncStream from mautrix.types import ( - BaseMessageEventContentFuncs, - Event, EventType, Filter, Membership, @@ -115,14 +111,11 @@ with open(args.meta, "r") as meta_file: if "/" in meta.main_class: module, main_class = meta.main_class.split("/", 1) else: - module = meta.modules[-1] + module = meta.modules[0] main_class = meta.main_class - -if args.meta != "maubot.yaml" and os.path.dirname(args.meta) != "": - sys.path.append(os.path.dirname(args.meta)) bot_module = importlib.import_module(module) plugin: type[Plugin] = getattr(bot_module, main_class) -loader = FileSystemLoader(os.path.dirname(args.meta), meta) +loader = FileSystemLoader(os.path.dirname(args.meta)) log.info(f"Initializing standalone {meta.id} v{meta.version} on maubot {__version__}") @@ -138,7 +131,6 @@ user_id = config["user.credentials.id"] device_id = config["user.credentials.device_id"] homeserver = config["user.credentials.homeserver"] access_token = config["user.credentials.access_token"] -appservice_listener = config["user.appservice"] crypto_store = state_store = None if device_id and not OlmMachine: @@ -196,10 +188,6 @@ if meta.webapp: resource = PrefixResource(web_base_path) resource.add_route(hdrs.METH_ANY, _handle_plugin_request) web_app.router.register_resource(resource) -elif appservice_listener: - web_app = web.Application() - web_runner = web.AppRunner(web_app, access_log_class=AccessLogger) - public_url = plugin_webapp = None else: web_app = web_runner = public_url = plugin_webapp = None @@ -207,31 +195,6 @@ loop = asyncio.get_event_loop() client: MaubotMatrixClient | None = None bot: Plugin | None = None -appservice: AppServiceServerMixin | None = None - - -if appservice_listener: - assert web_app is not None, "web_app is always set when appservice_listener is set" - appservice = AppServiceServerMixin( - ephemeral_events=True, - encryption_events=True, - log=logging.getLogger("maubot.appservice"), - hs_token=config["user.hs_token"], - ) - appservice.register_routes(web_app) - - @appservice.matrix_event_handler - async def handle_appservice_event(evt: Event) -> None: - if isinstance(evt.content, BaseMessageEventContentFuncs): - evt.content.trim_reply_fallback() - fake_sync_stream = SyncStream.JOINED_ROOM - if evt.type.is_ephemeral: - fake_sync_stream |= SyncStream.EPHEMERAL - else: - fake_sync_stream |= SyncStream.TIMELINE - setattr(evt, "source", fake_sync_stream) - tasks = client.dispatch_manual_event(evt.type, evt, include_global_handlers=True) - await asyncio.gather(*tasks) async def main(): @@ -254,8 +217,6 @@ async def main(): state_store=state_store, device_id=device_id, ) - if appservice: - client.api.as_user_id = user_id client.ignore_first_sync = config["user.ignore_first_sync"] client.ignore_initial_sync = config["user.ignore_initial_sync"] if crypto_store: @@ -264,11 +225,6 @@ async def main(): await crypto_store.open() client.crypto = OlmMachine(client, crypto_store, state_store) - if appservice: - appservice.otk_handler = client.crypto.handle_as_otk_counts - appservice.device_list_handler = client.crypto.handle_as_device_lists - appservice.to_device_handler = client.crypto.handle_as_to_device_event - client.api.as_device_id = device_id crypto_device_id = await crypto_store.get_device_id() if crypto_device_id and crypto_device_id != device_id: log.fatal( @@ -316,8 +272,6 @@ async def main(): ) await nb.put_filter_id(filter_id) _ = client.start(nb.filter_id) - elif appservice_listener and crypto_store and not client.crypto.account.shared: - await client.crypto.share_keys() if config["user.autojoin"]: log.debug("Autojoin is enabled") @@ -380,14 +334,9 @@ async def stop(suppress_stop_error: bool = False) -> None: except Exception: if not suppress_stop_error: log.exception("Error stopping bot") - if web_runner and web_runner.server: - try: - await web_runner.shutdown() - await web_runner.cleanup() - except RuntimeError: - if not suppress_stop_error: - await db.stop() - raise + if web_runner: + await web_runner.shutdown() + await web_runner.cleanup() await db.stop() @@ -398,10 +347,6 @@ signal.signal(signal.SIGTERM, signal.default_int_handler) try: log.info("Starting plugin") loop.run_until_complete(main()) -except SystemExit: - loop.run_until_complete(stop(suppress_stop_error=True)) - loop.close() - raise except (Exception, KeyboardInterrupt) as e: if isinstance(e, KeyboardInterrupt): log.info("Startup interrupted, stopping") diff --git a/maubot/standalone/config.py b/maubot/standalone/config.py index f2c90a0..ce17310 100644 --- a/maubot/standalone/config.py +++ b/maubot/standalone/config.py @@ -33,13 +33,9 @@ class Config(BaseFileConfig): copy("user.credentials.access_token") copy("user.credentials.device_id") copy("user.sync") - copy("user.appservice") - copy("user.hs_token") copy("user.autojoin") copy("user.displayname") copy("user.avatar_url") - copy("user.ignore_initial_sync") - copy("user.ignore_first_sync") if "server" in base: copy("server.hostname") copy("server.port") diff --git a/maubot/standalone/example-config.yaml b/maubot/standalone/example-config.yaml index 8ee4e43..1884b78 100644 --- a/maubot/standalone/example-config.yaml +++ b/maubot/standalone/example-config.yaml @@ -5,15 +5,9 @@ user: homeserver: https://example.com access_token: foo # If you want to enable encryption, set the device ID corresponding to the access token here. - # When using an appservice, you should use appservice login manually to generate a device ID and access token. device_id: null # Enable /sync? This is not needed for purely unencrypted webhook-based bots, but is necessary in most other cases. sync: true - # Receive appservice transactions? This will add a /_matrix/app/v1/transactions endpoint on - # the HTTP server configured below. The base_path will not be applied for the /transactions path. - appservice: false - # When appservice mode is enabled, the hs_token for the appservice. - hs_token: null # Automatically accept invites? autojoin: false # The displayname and avatar URL to set for the bot on startup. @@ -27,8 +21,7 @@ user: # if you want the bot to handle messages that were sent while the bot was down. ignore_first_sync: true -# Web server settings. These will only take effect if the plugin requests it using `webapp: true` in the meta file, -# or if user -> appservice is set to true. +# Web server settings. These will only take effect if the plugin requests it using `webapp: true` in the meta file. server: # The IP and port to listen to. hostname: 0.0.0.0 @@ -42,7 +35,7 @@ server: # The database for the plugin. Used for plugin data, the sync token and e2ee data (if enabled). # SQLite and Postgres are supported. -database: sqlite:bot.db +database: sqlite:///bot.db # Additional arguments for asyncpg.create_pool() or sqlite3.connect() # https://magicstack.github.io/asyncpg/current/api/index.html#asyncpg.pool.create_pool diff --git a/maubot/standalone/loader.py b/maubot/standalone/loader.py index 3d5a907..8c75c05 100644 --- a/maubot/standalone/loader.py +++ b/maubot/standalone/loader.py @@ -18,13 +18,12 @@ from __future__ import annotations import os import os.path -from ..loader import BasePluginLoader, PluginMeta +from ..loader import BasePluginLoader class FileSystemLoader(BasePluginLoader): - def __init__(self, path: str, meta: PluginMeta) -> None: + def __init__(self, path: str) -> None: self.path = path - self.meta = meta @property def source(self) -> str: diff --git a/maubot/testing/__init__.py b/maubot/testing/__init__.py deleted file mode 100644 index 1fcdfc0..0000000 --- a/maubot/testing/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# maubot - A plugin-based Matrix bot system. -# Copyright (C) 2023 Aurélien Bompard -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -from .bot import TestBot, make_message # noqa: F401 -from .fixtures import * # noqa: F401,F403 diff --git a/maubot/testing/bot.py b/maubot/testing/bot.py deleted file mode 100644 index 0519016..0000000 --- a/maubot/testing/bot.py +++ /dev/null @@ -1,100 +0,0 @@ -# maubot - A plugin-based Matrix bot system. -# Copyright (C) 2023 Aurélien Bompard -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -import asyncio -import time - -from attr import dataclass - -from maubot.matrix import MaubotMatrixClient, MaubotMessageEvent -from mautrix.api import HTTPAPI -from mautrix.types import ( - EventContent, - EventType, - MessageEvent, - MessageType, - RoomID, - TextMessageEventContent, -) - - -@dataclass -class MatrixEvent: - room_id: RoomID - event_type: EventType - content: EventContent - kwargs: dict - - -class TestBot: - """A mocked bot used for testing purposes. - - Send messages to the mock Matrix server with the ``send()`` method. - Look into the ``responded`` list to get what server has replied. - """ - - def __init__(self, mxid="@botname:example.com", mxurl="http://matrix.example.com"): - api = HTTPAPI(base_url=mxurl) - self.client = MaubotMatrixClient(api=api) - self.responded = [] - self.client.mxid = mxid - self.client.send_message_event = self._mock_send_message_event - - async def _mock_send_message_event(self, room_id, event_type, content, txn_id=None, **kwargs): - self.responded.append( - MatrixEvent(room_id=room_id, event_type=event_type, content=content, kwargs=kwargs) - ) - - async def dispatch(self, event_type: EventType, event): - tasks = self.client.dispatch_manual_event(event_type, event, force_synchronous=True) - return await asyncio.gather(*tasks) - - async def send( - self, - content, - html=None, - room_id="testroom", - msg_type=MessageType.TEXT, - sender="@dummy:example.com", - timestamp=None, - ): - event = make_message( - content, - html=html, - room_id=room_id, - msg_type=msg_type, - sender=sender, - timestamp=timestamp, - ) - await self.dispatch(EventType.ROOM_MESSAGE, MaubotMessageEvent(event, self.client)) - - -def make_message( - content, - html=None, - room_id="testroom", - msg_type=MessageType.TEXT, - sender="@dummy:example.com", - timestamp=None, -): - """Make a Matrix message event.""" - return MessageEvent( - type=EventType.ROOM_MESSAGE, - room_id=room_id, - event_id="test", - sender=sender, - timestamp=timestamp or int(time.time() * 1000), - content=TextMessageEventContent(msgtype=msg_type, body=content, formatted_body=html), - ) diff --git a/maubot/testing/fixtures.py b/maubot/testing/fixtures.py deleted file mode 100644 index e975782..0000000 --- a/maubot/testing/fixtures.py +++ /dev/null @@ -1,135 +0,0 @@ -# maubot - A plugin-based Matrix bot system. -# Copyright (C) 2023 Aurélien Bompard -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -from pathlib import Path -import asyncio -import logging - -from ruamel.yaml import YAML -import aiohttp -import pytest -import pytest_asyncio - -from maubot import Plugin -from maubot.loader import PluginMeta -from maubot.standalone.loader import FileSystemLoader -from mautrix.util.async_db import Database -from mautrix.util.config import BaseProxyConfig, RecursiveDict -from mautrix.util.logging import TraceLogger - -from .bot import TestBot - - -@pytest_asyncio.fixture -async def maubot_test_bot(): - return TestBot() - - -@pytest.fixture -def maubot_upgrade_table(): - return None - - -@pytest.fixture -def maubot_plugin_path(): - return Path(".") - - -@pytest.fixture -def maubot_plugin_meta(maubot_plugin_path): - yaml = YAML() - with open(maubot_plugin_path.joinpath("maubot.yaml")) as fh: - plugin_meta = PluginMeta.deserialize(yaml.load(fh.read())) - return plugin_meta - - -@pytest_asyncio.fixture -async def maubot_plugin_db(tmp_path, maubot_plugin_meta, maubot_upgrade_table): - if not maubot_plugin_meta.get("database", False): - return - db_path = tmp_path.joinpath("maubot-tests.db").as_posix() - db = Database.create( - f"sqlite:{db_path}", - upgrade_table=maubot_upgrade_table, - log=logging.getLogger("db"), - ) - await db.start() - yield db - await db.stop() - - -@pytest.fixture -def maubot_plugin_class(): - return Plugin - - -@pytest.fixture -def maubot_plugin_config_class(): - return BaseProxyConfig - - -@pytest.fixture -def maubot_plugin_config_dict(): - return {} - - -@pytest.fixture -def maubot_plugin_config_overrides(): - return {} - - -@pytest.fixture -def maubot_plugin_config( - maubot_plugin_path, - maubot_plugin_config_class, - maubot_plugin_config_dict, - maubot_plugin_config_overrides, -): - yaml = YAML() - with open(maubot_plugin_path.joinpath("base-config.yaml")) as fh: - base_config = RecursiveDict(yaml.load(fh)) - maubot_plugin_config_dict.update(maubot_plugin_config_overrides) - return maubot_plugin_config_class( - load=lambda: maubot_plugin_config_dict, - load_base=lambda: base_config, - save=lambda c: None, - ) - - -@pytest_asyncio.fixture -async def maubot_plugin( - maubot_test_bot, - maubot_plugin_db, - maubot_plugin_class, - maubot_plugin_path, - maubot_plugin_config, - maubot_plugin_meta, -): - loader = FileSystemLoader(maubot_plugin_path, maubot_plugin_meta) - async with aiohttp.ClientSession() as http: - instance = maubot_plugin_class( - client=maubot_test_bot.client, - loop=asyncio.get_running_loop(), - http=http, - instance_id="tests", - log=TraceLogger("test"), - config=maubot_plugin_config, - database=maubot_plugin_db, - webapp=None, - webapp_url=None, - loader=loader, - ) - await instance.internal_start() - yield instance diff --git a/optional-requirements.txt b/optional-requirements.txt index f5b378a..6d87db3 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -5,10 +5,3 @@ python-olm>=3,<4 pycryptodome>=3,<4 unpaddedbase64>=1,<3 - -#/testing -pytest -pytest-asyncio - -#/legacydb -SQLAlchemy>1,<1.4 diff --git a/pyproject.toml b/pyproject.toml index 6c2ca27..4cee457 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,5 +9,5 @@ skip = ["maubot/management/frontend"] [tool.black] line-length = 99 -target-version = ["py310"] +target-version = ["py38"] force-exclude = "maubot/management/frontend" diff --git a/requirements.txt b/requirements.txt index e1df001..c83d4db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,16 @@ -mautrix>=0.20.7,<0.21 +mautrix>=0.15.5,<0.16 aiohttp>=3,<4 yarl>=1,<2 -asyncpg>=0.20,<1 -aiosqlite>=0.16,<1 +SQLAlchemy>=1,<1.4 +asyncpg>=0.20,<0.26 +aiosqlite>=0.16,<0.18 commonmark>=0.9,<1 -ruamel.yaml>=0.15.35,<0.19 +ruamel.yaml>=0.15.35,<0.18 attrs>=18.1.0 -bcrypt>=3,<5 +bcrypt>=3,<4 packaging>=10 click>=7,<9 colorama>=0.4,<0.5 -questionary>=1,<3 +questionary>=1,<2 jinja2>=2,<4 -setuptools diff --git a/setup.py b/setup.py index 838196f..a4a16e1 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setuptools.setup( install_requires=install_requires, extras_require=extras_require, - python_requires="~=3.10", + python_requires="~=3.8", classifiers=[ "Development Status :: 4 - Beta", @@ -50,16 +50,13 @@ setuptools.setup( "Framework :: AsyncIO", "Programming Language :: Python", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", ], entry_points=""" [console_scripts] mbc=maubot.cli:app - [pytest11] - maubot=maubot.testing """, data_files=[ (".", ["maubot/example-config.yaml"]), @@ -71,7 +68,6 @@ setuptools.setup( "management/frontend/build/static/css/*", "management/frontend/build/static/js/*", "management/frontend/build/static/media/*", - "py.typed", ], "maubot.cli": ["res/*"], "maubot.standalone": ["example-config.yaml"],