From 5baab99957de118be3c969ebab359ef35fe66623 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 26 Apr 2022 23:44:52 +0300 Subject: [PATCH 01/90] Remove extra parameter. Fixes #165 --- maubot/cli/commands/logs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maubot/cli/commands/logs.py b/maubot/cli/commands/logs.py index 9a9c644..914f776 100644 --- a/maubot/cli/commands/logs.py +++ b/maubot/cli/commands/logs.py @@ -38,7 +38,7 @@ def logs(server: str, tail: int) -> None: global history_count history_count = tail loop = asyncio.get_event_loop() - future = asyncio.create_task(view_logs(server, token), loop=loop) + future = asyncio.create_task(view_logs(server, token)) try: loop.run_until_complete(future) except KeyboardInterrupt: From d4face01509169036ddb7f4c5751c48ba92607cf Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 26 Apr 2022 23:52:59 +0300 Subject: [PATCH 02/90] Actually fix mbc logs --- maubot/cli/commands/logs.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/maubot/cli/commands/logs.py b/maubot/cli/commands/logs.py index 914f776..e0ed07d 100644 --- a/maubot/cli/commands/logs.py +++ b/maubot/cli/commands/logs.py @@ -38,13 +38,7 @@ def logs(server: str, tail: int) -> None: global history_count history_count = tail loop = asyncio.get_event_loop() - future = asyncio.create_task(view_logs(server, token)) - try: - loop.run_until_complete(future) - except KeyboardInterrupt: - future.cancel() - loop.run_until_complete(future) - loop.close() + loop.run_until_complete(view_logs(server, token)) def parsedate(entry: Obj) -> None: From cc5f9141fcffc11d5cddb95f2d75603fea791e7e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 27 May 2022 22:47:58 +0300 Subject: [PATCH 03/90] Add more helpful error message --- maubot/cli/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/maubot/cli/config.py b/maubot/cli/config.py index a1adc2f..5fdc4ea 100644 --- a/maubot/cli/config.py +++ b/maubot/cli/config.py @@ -36,6 +36,7 @@ def get_default_server() -> tuple[str | None, str | None]: server = None if server is None: print(f"{Fore.RED}Default server not configured.{Fore.RESET}") + print(f"Perhaps you forgot to {Fore.CYAN}mbc login{Fore.RESET}?") return None, None return server, _get_token(server) From 0663b680ab548d35644d904657d3361b15daa7b8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 6 May 2022 15:40:07 +0300 Subject: [PATCH 04/90] Fix typo in pre commit config --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e1e07d5..508e4ae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,4 +17,4 @@ repos: rev: 5.10.1 hooks: - id: isort - files: ^maubot/.*\.pyi$ + files: ^maubot/.*\.pyi?$ From f74a67dd793a1799449846315659cc11a467c6cf Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 6 May 2022 16:18:45 +0300 Subject: [PATCH 05/90] Store instance database engine in database --- maubot/db/__init__.py | 4 +- maubot/db/instance.py | 40 +++++++++++--- maubot/db/upgrade/__init__.py | 2 +- .../upgrade/v02_instance_database_engine.py | 25 +++++++++ maubot/instance.py | 52 ++++++++++++++++--- maubot/loader/meta.py | 6 ++- .../frontend/src/pages/dashboard/Instance.js | 3 +- 7 files changed, 113 insertions(+), 19 deletions(-) create mode 100644 maubot/db/upgrade/v02_instance_database_engine.py diff --git a/maubot/db/__init__.py b/maubot/db/__init__.py index d6aeb09..68833ce 100644 --- a/maubot/db/__init__.py +++ b/maubot/db/__init__.py @@ -1,7 +1,7 @@ from mautrix.util.async_db import Database from .client import Client -from .instance import Instance +from .instance import DatabaseEngine, Instance from .upgrade import upgrade_table @@ -10,4 +10,4 @@ def init(db: Database) -> None: table.db = db -__all__ = ["upgrade_table", "init", "Client", "Instance"] +__all__ = ["upgrade_table", "init", "Client", "Instance", "DatabaseEngine"] diff --git a/maubot/db/instance.py b/maubot/db/instance.py index dff7064..5bb3f6a 100644 --- a/maubot/db/instance.py +++ b/maubot/db/instance.py @@ -16,6 +16,7 @@ from __future__ import annotations from typing import TYPE_CHECKING, ClassVar +from enum import Enum from asyncpg import Record from attr import dataclass @@ -26,6 +27,11 @@ from mautrix.util.async_db import Database fake_db = Database.create("") if TYPE_CHECKING else None +class DatabaseEngine(Enum): + SQLITE = "sqlite" + POSTGRES = "postgres" + + @dataclass class Instance: db: ClassVar[Database] = fake_db @@ -35,21 +41,31 @@ class Instance: enabled: bool primary_user: UserID config_str: str + database_engine: DatabaseEngine | None + + @property + def database_engine_str(self) -> str | None: + return self.database_engine.value if self.database_engine else None @classmethod def _from_row(cls, row: Record | None) -> Instance | None: if row is None: return None - return cls(**row) + data = {**row} + db_engine = data.pop("database_engine", None) + return cls(**data, database_engine=DatabaseEngine(db_engine) if db_engine else None) + + _columns = "id, type, enabled, primary_user, config, database_engine" @classmethod async def all(cls) -> list[Instance]: - rows = await cls.db.fetch("SELECT id, type, enabled, primary_user, config FROM instance") + q = f"SELECT {cls._columns} FROM instance" + rows = await cls.db.fetch(q) return [cls._from_row(row) for row in rows] @classmethod async def get(cls, id: str) -> Instance | None: - q = "SELECT id, type, enabled, primary_user, config FROM instance WHERE id=$1" + q = f"SELECT {cls._columns} FROM instance WHERE id=$1" return cls._from_row(await cls.db.fetchrow(q, id)) async def update_id(self, new_id: str) -> None: @@ -58,17 +74,27 @@ class Instance: @property def _values(self): - return self.id, self.type, self.enabled, self.primary_user, self.config_str + return ( + self.id, + self.type, + self.enabled, + self.primary_user, + self.config_str, + self.database_engine_str, + ) async def insert(self) -> None: q = ( - "INSERT INTO instance (id, type, enabled, primary_user, config) " - "VALUES ($1, $2, $3, $4, $5)" + "INSERT INTO instance (id, type, enabled, primary_user, config, database_engine) " + "VALUES ($1, $2, $3, $4, $5, $6)" ) await self.db.execute(q, *self._values) async def update(self) -> None: - q = "UPDATE instance SET type=$2, enabled=$3, primary_user=$4, config=$5 WHERE id=$1" + q = """ + UPDATE instance SET type=$2, enabled=$3, primary_user=$4, config=$5, database_engine=$6 + WHERE id=$1 + """ await self.db.execute(q, *self._values) async def delete(self) -> None: diff --git a/maubot/db/upgrade/__init__.py b/maubot/db/upgrade/__init__.py index 146e713..ed96422 100644 --- a/maubot/db/upgrade/__init__.py +++ b/maubot/db/upgrade/__init__.py @@ -2,4 +2,4 @@ from mautrix.util.async_db import UpgradeTable upgrade_table = UpgradeTable() -from . import v01_initial_revision +from . import v01_initial_revision, v02_instance_database_engine diff --git a/maubot/db/upgrade/v02_instance_database_engine.py b/maubot/db/upgrade/v02_instance_database_engine.py new file mode 100644 index 0000000..7d2d7e7 --- /dev/null +++ b/maubot/db/upgrade/v02_instance_database_engine.py @@ -0,0 +1,25 @@ +# maubot - A plugin-based Matrix bot system. +# Copyright (C) 2022 Tulir Asokan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from __future__ import annotations + +from mautrix.util.async_db import Connection + +from . import upgrade_table + + +@upgrade_table.register(description="Store instance database engine") +async def upgrade_v2(conn: Connection) -> None: + await conn.execute("ALTER TABLE instance ADD COLUMN database_engine TEXT") diff --git a/maubot/instance.py b/maubot/instance.py index b9b1c23..4d797e4 100644 --- a/maubot/instance.py +++ b/maubot/instance.py @@ -34,7 +34,7 @@ from mautrix.util.config import BaseProxyConfig, RecursiveDict from mautrix.util.logging import TraceLogger from .client import Client -from .db import Instance as DBInstance +from .db import DatabaseEngine, Instance as DBInstance from .lib.plugin_db import ProxyPostgresDatabase from .loader import DatabaseType, PluginLoader, ZippedPluginLoader from .plugin_base import Plugin @@ -71,10 +71,21 @@ class PluginInstance(DBInstance): started: bool def __init__( - self, id: str, type: str, enabled: bool, primary_user: UserID, config: str = "" + self, + id: str, + type: str, + enabled: bool, + primary_user: UserID, + config: str = "", + database_engine: DatabaseEngine | None = None, ) -> None: super().__init__( - id=id, type=type, enabled=bool(enabled), primary_user=primary_user, config_str=config + id=id, + type=type, + enabled=bool(enabled), + primary_user=primary_user, + config_str=config, + database_engine=database_engine, ) def __hash__(self) -> int: @@ -111,6 +122,8 @@ class PluginInstance(DBInstance): "database": ( self.inst_db is not None and self.maubot.config["api_features.instance_database"] ), + "database_interface": self.loader.meta.database_type_str if self.loader else "unknown", + "database_engine": self.database_engine_str, } def _introspect_sqlalchemy(self) -> dict: @@ -269,12 +282,27 @@ class PluginInstance(DBInstance): self, upgrade_table: UpgradeTable | None = None, actually_start: bool = True ) -> None: if self.loader.meta.database_type == DatabaseType.SQLALCHEMY: + if self.database_engine is None: + await self.update_db_engine(DatabaseEngine.SQLITE) + elif self.database_engine == DatabaseEngine.POSTGRES: + raise RuntimeError( + "Instance database engine is marked as Postgres, but plugin uses legacy " + "database interface, which doesn't support postgres." + ) self.inst_db = sql.create_engine(f"sqlite:///{self._sqlite_db_path}") elif self.loader.meta.database_type == DatabaseType.ASYNCPG: + if self.database_engine is None: + if os.path.exists(self._sqlite_db_path) or not self.maubot.plugin_postgres_db: + await self.update_db_engine(DatabaseEngine.SQLITE) + else: + await self.update_db_engine(DatabaseEngine.POSTGRES) instance_db_log = db_log.getChild(self.id) - # TODO should there be a way to choose between SQLite and Postgres - # for individual instances? Maybe checking the existence of the SQLite file. - if self.maubot.plugin_postgres_db: + if self.database_engine == DatabaseEngine.POSTGRES: + if not self.maubot.plugin_postgres_db: + raise RuntimeError( + "Instance database engine is marked as Postgres, but this maubot isn't " + "configured to support Postgres for plugin databases" + ) self.inst_db = ProxyPostgresDatabase( pool=self.maubot.plugin_postgres_db, instance_id=self.id, @@ -334,7 +362,12 @@ class PluginInstance(DBInstance): self.log.debug("Disabling webapp after plugin meta reload") self.disable_webapp() if self.loader.meta.database: - await self.start_database(cls.get_db_upgrade_table()) + try: + await self.start_database(cls.get_db_upgrade_table()) + except Exception: + self.log.exception("Failed to start instance database") + await self.update_enabled(False) + return config_class = cls.get_config_class() if config_class: try: @@ -455,6 +488,11 @@ class PluginInstance(DBInstance): self.enabled = enabled await self.update() + async def update_db_engine(self, db_engine: DatabaseEngine | None) -> None: + if db_engine is not None and db_engine != self.database_engine: + self.database_engine = db_engine + await self.update() + @classmethod @async_getter_lock async def get( diff --git a/maubot/loader/meta.py b/maubot/loader/meta.py index f16937b..d368e24 100644 --- a/maubot/loader/meta.py +++ b/maubot/loader/meta.py @@ -13,7 +13,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import List +from typing import List, Optional from attr import dataclass from packaging.version import InvalidVersion, Version @@ -63,3 +63,7 @@ class PluginMeta(SerializableAttrs): extra_files: List[str] = [] dependencies: List[str] = [] soft_dependencies: List[str] = [] + + @property + def database_type_str(self) -> Optional[str]: + return self.database_type.value if self.database else None diff --git a/maubot/management/frontend/src/pages/dashboard/Instance.js b/maubot/management/frontend/src/pages/dashboard/Instance.js index c487fb0..049c5e5 100644 --- a/maubot/management/frontend/src/pages/dashboard/Instance.js +++ b/maubot/management/frontend/src/pages/dashboard/Instance.js @@ -43,7 +43,7 @@ class Instance extends BaseMainView { } get entryKeys() { - return ["id", "primary_user", "enabled", "started", "type", "config"] + return ["id", "primary_user", "enabled", "started", "type", "config", "database_engine"] } get initialState() { @@ -54,6 +54,7 @@ class Instance extends BaseMainView { started: true, type: "", config: "", + database_engine: "", saving: false, deleting: false, From 6ec653c69f761887e63e811ba7da4a7af2e494bb Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 6 May 2022 19:16:29 +0300 Subject: [PATCH 06/90] Bump mautrix-python version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c83d4db..d15448c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -mautrix>=0.15.5,<0.16 +mautrix>=0.15.5,<0.17 aiohttp>=3,<4 yarl>=1,<2 SQLAlchemy>=1,<1.4 From e0daeeafc635f967a4af5153e230e7a3cc13f8de Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 19 Jun 2022 14:23:06 +0300 Subject: [PATCH 07/90] Add GitLab CI file used by all plugins --- .gitlab-ci-plugin.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .gitlab-ci-plugin.yml diff --git a/.gitlab-ci-plugin.yml b/.gitlab-ci-plugin.yml new file mode 100644 index 0000000..45ef06b --- /dev/null +++ b/.gitlab-ci-plugin.yml @@ -0,0 +1,29 @@ +image: dock.mau.dev/maubot/maubot + +stages: +- build + +variables: + PYTHONPATH: /opt/maubot + +build: + stage: build + except: + - tags + script: + - python3 -m maubot.cli build -o xyz.maubot.$CI_PROJECT_NAME-$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA.mbp + artifacts: + paths: + - "*.mbp" + expire_in: 365 days + +build tags: + stage: build + only: + - tags + script: + - python3 -m maubot.cli build -o xyz.maubot.$CI_PROJECT_NAME-$CI_COMMIT_TAG.mbp + artifacts: + paths: + - "*.mbp" + expire_in: never From 47d499dac8c51b9f7487795023e1f3444bd8bfe2 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 19 Jun 2022 14:24:16 +0300 Subject: [PATCH 08/90] Use PYTHONPATH instead of cd in docker mbc wrapper. Fixes #172 --- docker/mbc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/mbc.sh b/docker/mbc.sh index bffbd5e..5bde65a 100755 --- a/docker/mbc.sh +++ b/docker/mbc.sh @@ -1,3 +1,3 @@ #!/bin/sh -cd /opt/maubot +export PYTHONPATH=/opt/maubot python3 -m maubot.cli "$@" From 47d7e2b483aa598a601231b88bbd4af78f71e4c6 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 24 Jun 2022 18:21:53 +0300 Subject: [PATCH 09/90] Remove legacy alembic command in docker/run.sh --- docker/run.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/run.sh b/docker/run.sh index 2cd9b45..ff9b54c 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -30,7 +30,6 @@ mkdir -p /var/log/maubot /data/plugins /data/trash /data/dbs if [ ! -f /data/config.yaml ]; then cp example-config.yaml /data/config.yaml - # Apply some docker-specific adjustments to the config echo "Config file not found. Example config copied to /data/config.yaml" echo "Please modify the config file to your liking and restart the container." fixperms @@ -38,7 +37,6 @@ if [ ! -f /data/config.yaml ]; then exit fi -alembic -x config=/data/config.yaml upgrade head fixperms fixconfig if ls /data/plugins/*.db > /dev/null 2>&1; then From 4f4d7bc34284a518c886e424ac5a08524588642d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 10 Jul 2022 14:42:41 +0300 Subject: [PATCH 10/90] Update mautrix-python --- maubot/management/frontend/src/pages/dashboard/Log.js | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/maubot/management/frontend/src/pages/dashboard/Log.js b/maubot/management/frontend/src/pages/dashboard/Log.js index 5cd9833..b80e6ad 100644 --- a/maubot/management/frontend/src/pages/dashboard/Log.js +++ b/maubot/management/frontend/src/pages/dashboard/Log.js @@ -41,7 +41,7 @@ class LogEntry extends PureComponent { const req = this.props.line.matrix_http_request return <> - {req.method} {req.path} + {req.method} {req.url || req.path}
{Object.entries(req.content || {}).length > 0 && } diff --git a/requirements.txt b/requirements.txt index d15448c..6c45018 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -mautrix>=0.15.5,<0.17 +mautrix>=0.15.5,<0.18 aiohttp>=3,<4 yarl>=1,<2 SQLAlchemy>=1,<1.4 From 75574c267b9dc576d86048608cf9ddd49c9fbbe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian-Samuel=20Geb=C3=BChr?= Date: Mon, 11 Jul 2022 15:15:47 +0200 Subject: [PATCH 11/90] Add alertbot to plugin list --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 84317d1..77c344b 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ The plugin wishlist lives at . * [alternatingcaps](https://github.com/rom4nik/maubot-alternatingcaps) - A bot repeating last message using aLtErNaTiNg cApS. * [metric](https://github.com/edwardsdean/maubot_metric_bot) - A bot that will reply to a message that contains imperial units and replace them with metric units. * [urlpreview](https://github.com/coffeebank/coffee-maubot/tree/master/urlpreview) - A bot that responds to links with a link preview embed, using Matrix API to fetch meta tags +* [alertbot](https://github.com/moan0s/alertbot) - A bot that recives monitoring alerts via alertmanager and forwards them to a matrix room † Uses a synchronous library which can block the whole maubot process (e.g. requests instead of aiohttp) From 30e67111b1b7092e34920489e26fcb3853b54990 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 19 Jul 2022 12:44:05 +0200 Subject: [PATCH 12/90] Add autoreply to the list of third-party plugins --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 84317d1..a7f3691 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,8 @@ The plugin wishlist lives at . * [pocket](https://github.com/jaywink/maubot-pocket) - A bot integrating with Pocket to fetch articles and archive them. * [alternatingcaps](https://github.com/rom4nik/maubot-alternatingcaps) - A bot repeating last message using aLtErNaTiNg cApS. * [metric](https://github.com/edwardsdean/maubot_metric_bot) - A bot that will reply to a message that contains imperial units and replace them with metric units. -* [urlpreview](https://github.com/coffeebank/coffee-maubot/tree/master/urlpreview) - A bot that responds to links with a link preview embed, using Matrix API to fetch meta tags +* [urlpreview](https://github.com/coffeebank/coffee-maubot/tree/master/urlpreview) - A bot that responds to links with a link preview embed, using Matrix API to fetch meta tags. +* [autoreply](https://github.com/babolivier/maubot-autoreply) - A bot that sends automated replies when you're away, and shows you a summary of missed messages when you come back. † Uses a synchronous library which can block the whole maubot process (e.g. requests instead of aiohttp) From bfdd52502a4786d09cc551525f677f38664785f5 Mon Sep 17 00:00:00 2001 From: Trevor Bergeron Date: Thu, 21 Jul 2022 22:46:48 -0400 Subject: [PATCH 13/90] Fix must_consume_args --- maubot/handlers/command.py | 1 + 1 file changed, 1 insertion(+) diff --git a/maubot/handlers/command.py b/maubot/handlers/command.py index 495bc27..1d9cdfb 100644 --- a/maubot/handlers/command.py +++ b/maubot/handlers/command.py @@ -92,6 +92,7 @@ class CommandHandler: "get_name", "is_command_match", "require_subcommand", + "must_consume_args", "arg_fallthrough", "event_handler", "event_type", From 305faa05836e95388adb610baa6024f8a67ccec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Rie=C3=9F?= Date: Wed, 27 Jul 2022 13:45:30 +0200 Subject: [PATCH 14/90] Add bots to README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 84317d1..52d3480 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,8 @@ The plugin wishlist lives at . * [alternatingcaps](https://github.com/rom4nik/maubot-alternatingcaps) - A bot repeating last message using aLtErNaTiNg cApS. * [metric](https://github.com/edwardsdean/maubot_metric_bot) - A bot that will reply to a message that contains imperial units and replace them with metric units. * [urlpreview](https://github.com/coffeebank/coffee-maubot/tree/master/urlpreview) - A bot that responds to links with a link preview embed, using Matrix API to fetch meta tags +* [hasswebhookbot](https://github.com/v411e/hasswebhookbot) - A bot receiving webhooks from [Home Assistant](https://github.com/home-assistant) +* [ovgumensabot](https://github.com/v411e/ovgumensabot) - A bot that automatically sends meals from OvGU canteen every day † Uses a synchronous library which can block the whole maubot process (e.g. requests instead of aiohttp) From 1fe53b4c56640269ed314ed01f6eef7d775af6d0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 14 Aug 2022 18:51:12 +0300 Subject: [PATCH 15/90] Fix SSO login path --- maubot/management/api/client_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maubot/management/api/client_auth.py b/maubot/management/api/client_auth.py index c5baade..f630931 100644 --- a/maubot/management/api/client_auth.py +++ b/maubot/management/api/client_auth.py @@ -188,7 +188,7 @@ async def _do_sso(req: AuthRequestInfo) -> web.Response: / "client/auth_external_sso/complete" / waiter_id ) - sso_url = req.client.api.base_url.with_path(str(Path.login.sso.redirect)).with_query( + sso_url = req.client.api.base_url.with_path(str(Path.v3.login.sso.redirect)).with_query( {"redirectUrl": str(public_url)} ) sso_waiters[waiter_id] = req, asyncio.get_running_loop().create_future() From 33d7892e1321d415271b7cad64717f76c342ebfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Sch=C3=B6nberg?= Date: Sun, 14 Aug 2022 21:48:49 +0200 Subject: [PATCH 16/90] Fix choosing SPDX license during plugin init The keys used to be lower case, but were changed to mixed case in this commit: 068e268c632b90e5e4f2954f3eaf3aa9342c240c The identfier are now used as inputted by the user. --- maubot/cli/util/spdx.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maubot/cli/util/spdx.py b/maubot/cli/util/spdx.py index 10508b3..69f58b7 100644 --- a/maubot/cli/util/spdx.py +++ b/maubot/cli/util/spdx.py @@ -36,10 +36,10 @@ def load() -> None: def get(id: str) -> dict[str, str]: if not spdx_list: load() - return spdx_list[id.lower()] + return spdx_list[id] def valid(id: str) -> bool: if not spdx_list: load() - return id.lower() in spdx_list + return id in spdx_list From d574f84079c2469b71e6c09c4b804e8bf6156153 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 20 Aug 2022 14:37:31 +0300 Subject: [PATCH 17/90] Fix saving plugin configs --- maubot/instance.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/maubot/instance.py b/maubot/instance.py index 4d797e4..895c700 100644 --- a/maubot/instance.py +++ b/maubot/instance.py @@ -276,7 +276,11 @@ class PluginInstance(DBInstance): def save_config(self, data: RecursiveDict[CommentedMap]) -> None: buf = io.StringIO() yaml.dump(data, buf) - self.config_str = buf.getvalue() + val = buf.getvalue() + if val != self.config_str: + self.config_str = val + self.log.debug("Creating background task to save updated config") + asyncio.create_task(self.update()) async def start_database( self, upgrade_table: UpgradeTable | None = None, actually_start: bool = True From fbacad8676a8e87ffbf6fe564ef65659c21801ab Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 12 Sep 2022 21:22:33 +0300 Subject: [PATCH 18/90] Add shortcut for redact in MaubotMessageEvent --- maubot/matrix.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/maubot/matrix.py b/maubot/matrix.py index 9c90c8f..a66ebfe 100644 --- a/maubot/matrix.py +++ b/maubot/matrix.py @@ -123,6 +123,9 @@ class MaubotMessageEvent(MessageEvent): def react(self, key: str) -> Awaitable[EventID]: return self.client.react(self.room_id, self.event_id, key) + def redact(self, reason: str | None = None) -> Awaitable[EventID]: + return self.client.redact(self.room_id, self.event_id, reason=reason) + def edit( self, content: str | MessageEventContent, From 790488823359ccd3b031e8994a22c88a4c124e7b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 10 Oct 2022 19:01:56 +0300 Subject: [PATCH 19/90] Update mautrix-python and asyncpg --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6c45018..8cce673 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -mautrix>=0.15.5,<0.18 +mautrix>=0.15.5,<0.19 aiohttp>=3,<4 yarl>=1,<2 SQLAlchemy>=1,<1.4 -asyncpg>=0.20,<0.26 +asyncpg>=0.20,<0.27 aiosqlite>=0.16,<0.18 commonmark>=0.9,<1 ruamel.yaml>=0.15.35,<0.18 From cab88d153b3b85b39ab198f65f2d2d185040a313 Mon Sep 17 00:00:00 2001 From: yoxcu Date: Tue, 18 Oct 2022 02:45:30 +0200 Subject: [PATCH 20/90] Added token bot to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 84317d1..7d2f218 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ The plugin wishlist lives at . * [alternatingcaps](https://github.com/rom4nik/maubot-alternatingcaps) - A bot repeating last message using aLtErNaTiNg cApS. * [metric](https://github.com/edwardsdean/maubot_metric_bot) - A bot that will reply to a message that contains imperial units and replace them with metric units. * [urlpreview](https://github.com/coffeebank/coffee-maubot/tree/master/urlpreview) - A bot that responds to links with a link preview embed, using Matrix API to fetch meta tags +* [token](https://github.com/yoxcu/maubot-token) - A maubot to create and manage your synapse user registration tokens. † Uses a synchronous library which can block the whole maubot process (e.g. requests instead of aiohttp) From 6f20151a8905b63f64988541c33367628d2b3dab Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 25 Oct 2022 11:46:05 +0200 Subject: [PATCH 21/90] Add redactbot Add 3rd party redactbot --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 84317d1..b5ed7d2 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ The plugin wishlist lives at . * [alternatingcaps](https://github.com/rom4nik/maubot-alternatingcaps) - A bot repeating last message using aLtErNaTiNg cApS. * [metric](https://github.com/edwardsdean/maubot_metric_bot) - A bot that will reply to a message that contains imperial units and replace them with metric units. * [urlpreview](https://github.com/coffeebank/coffee-maubot/tree/master/urlpreview) - A bot that responds to links with a link preview embed, using Matrix API to fetch meta tags +* [redactbot](https://gitlab.com/sspaeth/redactbot) - A bot that immediately redacts any posted file (except for whitelisted types). † Uses a synchronous library which can block the whole maubot process (e.g. requests instead of aiohttp) From a21b106c71e1bcf61f7e28e1eaeba3f2cb8281d2 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 5 Nov 2022 23:51:24 +0200 Subject: [PATCH 22/90] Update changelog [skip ci] --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52d4044..62ca1b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# v0.3.2 (unreleased) + +* Added `evt.redact()` as a shortcut for `self.client.redact(evt.room_id, evt.event_id)`. +* Fixed `mbc logs` command not working on Python 3.8+. +* Fixed saving plugin configs (broke in v0.3.0). +* Fixed SSO login using the wrong API path (probably broke in v0.3.0). +* Stopped using `cd` in the docker image's `mbc` wrapper to enable using + path-dependent commands like `mbc build` by mounting a directory. + # v0.3.1 (2022-03-29) * Added encryption dependencies to standalone dockerfile. From 7ec58c6347605c6ff4c4356175e2cc147d15acc0 Mon Sep 17 00:00:00 2001 From: William Kray Date: Tue, 20 Dec 2022 12:20:56 -0800 Subject: [PATCH 23/90] add several of my plugins --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 7b20895..14f165f 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,14 @@ The plugin wishlist lives at . * [ovgumensabot](https://github.com/v411e/ovgumensabot) - A bot that automatically sends meals from OvGU canteen every day. * [token](https://github.com/yoxcu/maubot-token) - A maubot to create and manage your synapse user registration tokens. * [redactbot](https://gitlab.com/sspaeth/redactbot) - A bot that immediately redacts any posted file (except for whitelisted types). +* [join](https://github.com/williamkray/maubot-join) - A plugin that restricts who can convince your bot to join new rooms to certain users. +* [create-room](https://github.com/williamkray/maubot-createroom) - A plugin that creates new rooms and automatically + sets them to be part of a private Matrix Space. +* [welcome](https://github.com/williamkray/maubot-welcome) - A plugin that greets new people with a configurable message when they join a room. +* [activity tracker](https://github.com/williamkray/maubot-kickbot) - A plugin that minimally tracks user activity + within a space. Useful for kicking inactive users from a private community. +* [random subreddit post](https://github.com/williamkray/maubot-reddit) - A plugin that returns a random post from a + given subreddit. † Uses a synchronous library which can block the whole maubot process (e.g. requests instead of aiohttp) From 535b6672a6d940e41063debbf41b8352018f2caf Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 29 Dec 2022 18:33:15 +0200 Subject: [PATCH 24/90] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 14f165f..6dbce90 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ The plugin wishlist lives at . * [alertbot](https://github.com/moan0s/alertbot) - A bot that recives monitoring alerts via alertmanager and forwards them to a matrix room. * [hasswebhookbot](https://github.com/v411e/hasswebhookbot) - A bot receiving webhooks from [Home Assistant](https://github.com/home-assistant). * [ovgumensabot](https://github.com/v411e/ovgumensabot) - A bot that automatically sends meals from OvGU canteen every day. -* [token](https://github.com/yoxcu/maubot-token) - A maubot to create and manage your synapse user registration tokens. +* †[token](https://github.com/yoxcu/maubot-token) - A maubot to create and manage your synapse user registration tokens. * [redactbot](https://gitlab.com/sspaeth/redactbot) - A bot that immediately redacts any posted file (except for whitelisted types). * [join](https://github.com/williamkray/maubot-join) - A plugin that restricts who can convince your bot to join new rooms to certain users. * [create-room](https://github.com/williamkray/maubot-createroom) - A plugin that creates new rooms and automatically From 6fd8f7ed00021048a638e16caeb4ae5a47286f24 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 31 Dec 2022 02:28:10 +0200 Subject: [PATCH 25/90] Update dependencies and alpine --- .gitlab-ci.yml | 2 +- Dockerfile | 14 ++++++-------- Dockerfile.ci | 2 +- dev-requirements.txt | 2 +- requirements.txt | 6 +++--- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 797e095..0c5196e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,7 @@ default: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY build frontend: - image: node:16-alpine + image: node:18-alpine stage: build frontend before_script: [] variables: diff --git a/Dockerfile b/Dockerfile index 8b7eae7..656a7c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ -FROM node:16 AS frontend-builder +FROM node:18 AS frontend-builder COPY ./maubot/management/frontend /frontend RUN cd /frontend && yarn --prod && yarn build -FROM alpine:3.15 +FROM alpine:3.17 RUN apk add --no-cache \ python3 py3-pip py3-setuptools py3-wheel \ @@ -21,11 +21,10 @@ RUN apk add --no-cache \ py3-packaging \ py3-markdown \ py3-alembic \ -# py3-cssselect \ + py3-cssselect \ py3-commonmark \ py3-pygments \ py3-tz \ -# py3-tzlocal \ py3-regex \ py3-wcwidth \ # encryption @@ -39,9 +38,8 @@ RUN apk add --no-cache \ py3-magic \ py3-feedparser \ py3-dateutil \ - py3-lxml -# py3-gitlab -# py3-semver@edge + py3-lxml \ + py3-semver # TODO remove pillow, magic, feedparser, lxml, gitlab and semver when maubot supports installing dependencies COPY requirements.txt /opt/maubot/requirements.txt @@ -49,7 +47,7 @@ COPY optional-requirements.txt /opt/maubot/optional-requirements.txt WORKDIR /opt/maubot RUN apk add --virtual .build-deps python3-dev build-base git \ && pip3 install -r requirements.txt -r optional-requirements.txt \ - dateparser langdetect python-gitlab pyquery cchardet semver tzlocal cssselect \ + dateparser langdetect python-gitlab pyquery tzlocal \ && apk del .build-deps # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies diff --git a/Dockerfile.ci b/Dockerfile.ci index 8c0f49b..d3a7e32 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM alpine:3.15 +FROM alpine:3.17 RUN apk add --no-cache \ python3 py3-pip py3-setuptools py3-wheel \ diff --git a/dev-requirements.txt b/dev-requirements.txt index 16231f3..14b83e9 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,3 @@ pre-commit>=2.10.1,<3 isort>=5.10.1,<6 -black>=22.3.0,<22 +black>=22.3.0,<23 diff --git a/requirements.txt b/requirements.txt index 8cce673..d050f78 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,12 +2,12 @@ mautrix>=0.15.5,<0.19 aiohttp>=3,<4 yarl>=1,<2 SQLAlchemy>=1,<1.4 -asyncpg>=0.20,<0.27 -aiosqlite>=0.16,<0.18 +asyncpg>=0.20,<0.28 +aiosqlite>=0.16,<0.19 commonmark>=0.9,<1 ruamel.yaml>=0.15.35,<0.18 attrs>=18.1.0 -bcrypt>=3,<4 +bcrypt>=3,<5 packaging>=10 click>=7,<9 From e99a13a391bc55ab3c78411b28249182d6016940 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 5 Jan 2023 00:27:29 +0200 Subject: [PATCH 26/90] Drop support for custom API paths Changing the base public URL is still possible (which may be useful if someone wants to use a reverse proxy without adding subdomains). Fixes #195 --- maubot/config.py | 2 -- maubot/example-config.yaml | 4 ---- maubot/management/api/client_auth.py | 3 +-- maubot/management/api/middleware.py | 2 +- maubot/management/frontend/src/pages/Main.js | 5 ++--- maubot/server.py | 10 +++------- 6 files changed, 7 insertions(+), 19 deletions(-) diff --git a/maubot/config.py b/maubot/config.py index e11fe1c..c5368cd 100644 --- a/maubot/config.py +++ b/maubot/config.py @@ -52,11 +52,9 @@ class Config(BaseFileConfig): copy("server.port") copy("server.public_url") copy("server.listen") - copy("server.base_path") copy("server.ui_base_path") copy("server.plugin_base_path") copy("server.override_resource_path") - copy("server.appservice_base_path") shared_secret = self["server.unshared_secret"] if shared_secret is None or shared_secret == "generate": base["server.unshared_secret"] = self._new_token() diff --git a/maubot/example-config.yaml b/maubot/example-config.yaml index d157269..0181846 100644 --- a/maubot/example-config.yaml +++ b/maubot/example-config.yaml @@ -55,8 +55,6 @@ server: port: 29316 # Public base URL where the server is visible. public_url: https://example.com - # The base management API path. - base_path: /_matrix/maubot/v1 # The base path for the UI. ui_base_path: /_matrix/maubot # The base path for plugin endpoints. The instance ID will be appended directly. @@ -64,8 +62,6 @@ server: # Override path from where to load UI resources. # Set to false to using pkg_resources to find the path. override_resource_path: false - # The base appservice API path. Use / for legacy appservice API and /_matrix/app/v1 for v1. - appservice_base_path: /_matrix/app/v1 # The shared secret to sign API access tokens. # Set to "generate" to generate and save a new token at startup. unshared_secret: generate diff --git a/maubot/management/api/client_auth.py b/maubot/management/api/client_auth.py index f630931..4e5e201 100644 --- a/maubot/management/api/client_auth.py +++ b/maubot/management/api/client_auth.py @@ -184,8 +184,7 @@ async def _do_sso(req: AuthRequestInfo) -> web.Response: cfg = get_config() public_url = ( URL(cfg["server.public_url"]) - / cfg["server.base_path"].lstrip("/") - / "client/auth_external_sso/complete" + / "_matrix/maubot/v1/client/auth_external_sso/complete" / waiter_id ) sso_url = req.client.api.base_url.with_path(str(Path.v3.login.sso.redirect)).with_query( diff --git a/maubot/management/api/middleware.py b/maubot/management/api/middleware.py index 0ecb681..17141fa 100644 --- a/maubot/management/api/middleware.py +++ b/maubot/management/api/middleware.py @@ -29,7 +29,7 @@ log = logging.getLogger("maubot.server") @web.middleware async def auth(request: web.Request, handler: Handler) -> web.Response: - subpath = request.path[len(get_config()["server.base_path"]) :] + subpath = request.path[len("/_matrix/maubot/v1") :] if ( subpath.startswith("/auth/") or subpath.startswith("/client/auth_external_sso/complete/") diff --git a/maubot/management/frontend/src/pages/Main.js b/maubot/management/frontend/src/pages/Main.js index fb650b6..5ee374e 100644 --- a/maubot/management/frontend/src/pages/Main.js +++ b/maubot/management/frontend/src/pages/Main.js @@ -45,9 +45,8 @@ class Main extends Component { const resp = await fetch(process.env.PUBLIC_URL + "/paths.json", { headers: { "Content-Type": "application/json" }, }) - const apiPathJson = await resp.json() - const apiPath = apiPathJson.api_path - api.setBasePath(`${apiPath}`) + const apiPaths = await resp.json() + api.setBasePath(apiPaths.api_path) } catch (err) { console.error("Failed to get API path:", err) } diff --git a/maubot/server.py b/maubot/server.py index d70ae7e..ac3132d 100644 --- a/maubot/server.py +++ b/maubot/server.py @@ -52,7 +52,7 @@ class MaubotServer: self.config = config self.setup_appservice() - self.app.add_subapp(config["server.base_path"], management_api) + self.app.add_subapp("/_matrix/maubot/v1", management_api) self.setup_instance_subapps() self.setup_management_ui() @@ -93,7 +93,7 @@ class MaubotServer: self.app.router.register_resource(resource) def setup_appservice(self) -> None: - as_path = PathBuilder(self.config["server.appservice_base_path"]) + as_path = PathBuilder("/_matrix/appservice/v1") self.add_route(Method.PUT, as_path.transactions, self.handle_transaction) def setup_management_ui(self) -> None: @@ -140,16 +140,12 @@ class MaubotServer: f"{ui_base}/{file}", lambda _: web.Response(body=data, content_type=mime) ) - # also set up a resource path for the public url path prefix config - # cut the prefix path from public_url public_url = self.config["server.public_url"] - base_path = self.config["server.base_path"] public_url_path = "" if public_url: public_url_path = URL(public_url).path.rstrip("/") - # assemble with base_path - api_path = f"{public_url_path}{base_path}" + api_path = f"{public_url_path}/_matrix/maubot/v1" path_prefix_response_body = json.dumps({"api_path": api_path.rstrip("/")}) self.app.router.add_get( From ada4978879772b4350467af5f5f58071f6310d3f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 5 Jan 2023 00:31:05 +0200 Subject: [PATCH 27/90] Update changelog --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62ca1b3..ec50b4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,16 @@ -# v0.3.2 (unreleased) +# v0.4.0 (unreleased) +* Dropped support for using a custom maubot API base path. + * The public URL can still have a path prefix, e.g. when using a reverse + proxy. Both the web interface and `mbc` CLI tool should work fine with + custom prefixes. * Added `evt.redact()` as a shortcut for `self.client.redact(evt.room_id, evt.event_id)`. * Fixed `mbc logs` command not working on Python 3.8+. * Fixed saving plugin configs (broke in v0.3.0). * Fixed SSO login using the wrong API path (probably broke in v0.3.0). * Stopped using `cd` in the docker image's `mbc` wrapper to enable using path-dependent commands like `mbc build` by mounting a directory. +* Updated Docker image to Alpine 3.17. # v0.3.1 (2022-03-29) From f87e4ca7b9f97fe592e083c6150218890fa51326 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 5 Jan 2023 00:37:47 +0200 Subject: [PATCH 28/90] Update gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 99c9d7d..c57bb3c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,8 @@ pip-selfcheck.json *.pyc __pycache__ -*.db +*.db* +*.log /*.yaml !example-config.yaml !.pre-commit-config.yaml From bf3a3b65d839cf78ab03df39b2b80f24eb481843 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 29 Jan 2023 17:13:52 +0200 Subject: [PATCH 29/90] Bump version to 0.4.0 --- CHANGELOG.md | 2 +- maubot/__meta__.py | 2 +- setup.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec50b4d..20cb0bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# v0.4.0 (unreleased) +# v0.4.0 (2023-01-29) * Dropped support for using a custom maubot API base path. * The public URL can still have a path prefix, e.g. when using a reverse diff --git a/maubot/__meta__.py b/maubot/__meta__.py index 260c070..6a9beea 100644 --- a/maubot/__meta__.py +++ b/maubot/__meta__.py @@ -1 +1 @@ -__version__ = "0.3.1" +__version__ = "0.4.0" diff --git a/setup.py b/setup.py index a4a16e1..51687cc 100644 --- a/setup.py +++ b/setup.py @@ -53,6 +53,7 @@ setuptools.setup( "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], entry_points=""" [console_scripts] From f0e2bb3d628cffa5a254d445dea894cfd0bdcf49 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 31 Jan 2023 16:35:56 +0200 Subject: [PATCH 30/90] Add option to use thread for replies --- maubot/matrix.py | 11 ++++++++++- requirements.txt | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/maubot/matrix.py b/maubot/matrix.py index a66ebfe..31efdfc 100644 --- a/maubot/matrix.py +++ b/maubot/matrix.py @@ -82,6 +82,7 @@ class MaubotMessageEvent(MessageEvent): markdown: bool = True, allow_html: bool = False, reply: bool | str = False, + reply_in_thread: bool = False, edits: EventID | MessageEvent | None = None, ) -> EventID: if isinstance(content, str): @@ -102,6 +103,8 @@ class MaubotMessageEvent(MessageEvent): f"{self.sender}" f": {fmt_body}" ) + elif reply_in_thread: + content.set_thread_parent(self) else: content.set_reply(self) return await self.client.send_message_event(self.room_id, event_type, content) @@ -112,9 +115,15 @@ class MaubotMessageEvent(MessageEvent): event_type: EventType = EventType.ROOM_MESSAGE, markdown: bool = True, allow_html: bool = False, + thread: bool = False, ) -> Awaitable[EventID]: return self.respond( - content, event_type, markdown=markdown, reply=True, allow_html=allow_html + content, + event_type, + markdown=markdown, + reply=True, + reply_in_thread=thread, + allow_html=allow_html, ) def mark_read(self) -> Awaitable[None]: diff --git a/requirements.txt b/requirements.txt index d050f78..c0cada8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -mautrix>=0.15.5,<0.19 +mautrix>=0.16.2,<0.20 aiohttp>=3,<4 yarl>=1,<2 SQLAlchemy>=1,<1.4 From 41cbf6d7886f91cacfc9424109bdf39899d5ffc1 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 5 Feb 2023 22:34:54 +0200 Subject: [PATCH 31/90] Expose plugin metadata in standalone mode loader --- maubot/standalone/__main__.py | 2 +- maubot/standalone/loader.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/maubot/standalone/__main__.py b/maubot/standalone/__main__.py index 40a218d..87adc35 100644 --- a/maubot/standalone/__main__.py +++ b/maubot/standalone/__main__.py @@ -115,7 +115,7 @@ else: main_class = meta.main_class bot_module = importlib.import_module(module) plugin: type[Plugin] = getattr(bot_module, main_class) -loader = FileSystemLoader(os.path.dirname(args.meta)) +loader = FileSystemLoader(os.path.dirname(args.meta), meta) log.info(f"Initializing standalone {meta.id} v{meta.version} on maubot {__version__}") diff --git a/maubot/standalone/loader.py b/maubot/standalone/loader.py index 8c75c05..3d5a907 100644 --- a/maubot/standalone/loader.py +++ b/maubot/standalone/loader.py @@ -18,12 +18,13 @@ from __future__ import annotations import os import os.path -from ..loader import BasePluginLoader +from ..loader import BasePluginLoader, PluginMeta class FileSystemLoader(BasePluginLoader): - def __init__(self, path: str) -> None: + def __init__(self, path: str, meta: PluginMeta) -> None: self.path = path + self.meta = meta @property def source(self) -> str: From 955fb2723ec3bb8e1adc9632b4347c6c299b98e8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 5 Feb 2023 23:09:07 +0200 Subject: [PATCH 32/90] Update black --- .pre-commit-config.yaml | 4 ++-- maubot/lib/zipimport.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 508e4ae..4db1291 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,13 +8,13 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 23.1.0 hooks: - id: black language_version: python3 files: ^maubot/.*\.pyi?$ - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort files: ^maubot/.*\.pyi?$ diff --git a/maubot/lib/zipimport.py b/maubot/lib/zipimport.py index 963f14f..e7b77db 100644 --- a/maubot/lib/zipimport.py +++ b/maubot/lib/zipimport.py @@ -323,6 +323,7 @@ _zip_searchorder = ( (".py", False, False), ) + # Given a module name, return the potential file path in the # archive (without extension). def _get_module_path(self, fullname): @@ -351,6 +352,7 @@ def _get_module_info(self, fullname): # implementation + # _read_directory(archive) -> files dict (new reference) # # Given a path to a Zip archive, build a dict, mapping file names @@ -524,6 +526,7 @@ cp437_table = ( _importing_zlib = False + # Return the zlib.decompress function object, or NULL if zlib couldn't # be imported. The function is cached when found, so subsequent calls # don't import zlib again. From c0c7b58a3f80962f173759a3df5588e058602c6f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 12:45:48 +0200 Subject: [PATCH 33/90] Update linters --- .github/workflows/python-lint.yml | 2 +- .pre-commit-config.yaml | 2 +- dev-requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index 05a8bd2..d88b8a8 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -9,7 +9,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: - python-version: "3.10" + python-version: "3.11" - uses: isort/isort-action@master with: sortPaths: "./maubot" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4db1291..0de079e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.4.0 hooks: - id: trailing-whitespace exclude_types: [markdown] diff --git a/dev-requirements.txt b/dev-requirements.txt index 14b83e9..5cd14c2 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,3 @@ pre-commit>=2.10.1,<3 isort>=5.10.1,<6 -black>=22.3.0,<23 +black>=23,<24 From e73869bb194ed6b94f1760b2f225c308160ff3ec Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 12:51:19 +0200 Subject: [PATCH 34/90] Use new wrapper for creating background tasks --- maubot/client.py | 5 +++-- maubot/instance.py | 3 ++- maubot/management/api/log.py | 4 +++- requirements.txt | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/maubot/client.py b/maubot/client.py index 81d3d17..99816f6 100644 --- a/maubot/client.py +++ b/maubot/client.py @@ -15,7 +15,7 @@ # along with this program. If not, see . from __future__ import annotations -from typing import TYPE_CHECKING, Any, AsyncGenerator, AsyncIterable, Awaitable, Callable, cast +from typing import TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable, cast from collections import defaultdict import asyncio import logging @@ -43,6 +43,7 @@ from mautrix.types import ( ) from mautrix.util.async_getter_lock import async_getter_lock from mautrix.util.logging import TraceLogger +from mautrix.util import background_task from .db import Client as DBClient from .matrix import MaubotMatrixClient @@ -254,7 +255,7 @@ class Client(DBClient): self.log.warning( f"Failed to get /account/whoami, retrying in {(try_n + 1) * 10}s: {e}" ) - _ = asyncio.create_task(self.start(try_n + 1)) + background_task.create(self.start(try_n + 1)) return if whoami.user_id != self.id: self.log.error(f"User ID mismatch: expected {self.id}, but got {whoami.user_id}") diff --git a/maubot/instance.py b/maubot/instance.py index 895c700..67b82b5 100644 --- a/maubot/instance.py +++ b/maubot/instance.py @@ -32,6 +32,7 @@ from mautrix.util.async_db import Database, Scheme, UpgradeTable from mautrix.util.async_getter_lock import async_getter_lock from mautrix.util.config import BaseProxyConfig, RecursiveDict from mautrix.util.logging import TraceLogger +from mautrix.util import background_task from .client import Client from .db import DatabaseEngine, Instance as DBInstance @@ -280,7 +281,7 @@ class PluginInstance(DBInstance): if val != self.config_str: self.config_str = val self.log.debug("Creating background task to save updated config") - asyncio.create_task(self.update()) + background_task.create(self.update()) async def start_database( self, upgrade_table: UpgradeTable | None = None, actually_start: bool = True diff --git a/maubot/management/api/log.py b/maubot/management/api/log.py index 1f0b4bb..14c80cd 100644 --- a/maubot/management/api/log.py +++ b/maubot/management/api/log.py @@ -22,6 +22,8 @@ import logging from aiohttp import web, web_ws +from mautrix.util import background_task + from .auth import is_valid_token from .base import routes @@ -142,7 +144,7 @@ async def log_websocket(request: web.Request) -> web.WebSocketResponse: await ws.close(code=4000) log.debug(f"Connection from {request.remote} terminated due to no authentication") - asyncio.create_task(close_if_not_authenticated()) + background_task.create(close_if_not_authenticated()) try: msg: web_ws.WSMessage diff --git a/requirements.txt b/requirements.txt index c0cada8..50526b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -mautrix>=0.16.2,<0.20 +mautrix>=0.19.4,<0.20 aiohttp>=3,<4 yarl>=1,<2 SQLAlchemy>=1,<1.4 From 715659485819406cd8267863a44c797d8da33726 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 12:53:19 +0200 Subject: [PATCH 35/90] Fix import order --- maubot/client.py | 2 +- maubot/instance.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/maubot/client.py b/maubot/client.py index 99816f6..bdb76fc 100644 --- a/maubot/client.py +++ b/maubot/client.py @@ -41,9 +41,9 @@ from mautrix.types import ( SyncToken, UserID, ) +from mautrix.util import background_task from mautrix.util.async_getter_lock import async_getter_lock from mautrix.util.logging import TraceLogger -from mautrix.util import background_task from .db import Client as DBClient from .matrix import MaubotMatrixClient diff --git a/maubot/instance.py b/maubot/instance.py index 67b82b5..c63e5ab 100644 --- a/maubot/instance.py +++ b/maubot/instance.py @@ -28,11 +28,11 @@ from ruamel.yaml.comments import CommentedMap import sqlalchemy as sql from mautrix.types import UserID +from mautrix.util import background_task from mautrix.util.async_db import Database, Scheme, UpgradeTable from mautrix.util.async_getter_lock import async_getter_lock from mautrix.util.config import BaseProxyConfig, RecursiveDict from mautrix.util.logging import TraceLogger -from mautrix.util import background_task from .client import Client from .db import DatabaseEngine, Instance as DBInstance From def923d4448433b90418561cd0fee281b53ed1de Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 14 Feb 2023 21:59:49 +0200 Subject: [PATCH 36/90] Adjust thread behavior when responding --- maubot/matrix.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/maubot/matrix.py b/maubot/matrix.py index 31efdfc..2387617 100644 --- a/maubot/matrix.py +++ b/maubot/matrix.py @@ -24,6 +24,7 @@ import attr from mautrix.client import Client as MatrixClient, SyncStream from mautrix.errors import DecryptionError from mautrix.types import ( + BaseMessageEventContentFuncs, EncryptedEvent, Event, EventID, @@ -82,7 +83,7 @@ class MaubotMessageEvent(MessageEvent): markdown: bool = True, allow_html: bool = False, reply: bool | str = False, - reply_in_thread: bool = False, + in_thread: bool | None = None, edits: EventID | MessageEvent | None = None, ) -> EventID: if isinstance(content, str): @@ -94,7 +95,19 @@ class MaubotMessageEvent(MessageEvent): ) if edits: content.set_edit(edits) - elif reply: + if ( + not edits + and in_thread is not False + and ( + in_thread + or ( + isinstance(self.content, BaseMessageEventContentFuncs) + and self.content.get_thread_parent() + ) + ) + ): + content.set_thread_parent(self) + if reply and not edits: if reply != "force" and self.disable_reply: content.body = f"{self.sender}: {content.body}" fmt_body = content.formatted_body or escape(content.body).replace("\n", "
") @@ -103,8 +116,6 @@ class MaubotMessageEvent(MessageEvent): f"{self.sender}" f": {fmt_body}" ) - elif reply_in_thread: - content.set_thread_parent(self) else: content.set_reply(self) return await self.client.send_message_event(self.room_id, event_type, content) @@ -115,14 +126,14 @@ class MaubotMessageEvent(MessageEvent): event_type: EventType = EventType.ROOM_MESSAGE, markdown: bool = True, allow_html: bool = False, - thread: bool = False, + in_thread: bool | None = None, ) -> Awaitable[EventID]: return self.respond( content, event_type, markdown=markdown, reply=True, - reply_in_thread=thread, + in_thread=in_thread, allow_html=allow_html, ) From d4bb502def3999a76a142dcc16924acbb5e35bfd Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 14 Feb 2023 22:07:40 +0200 Subject: [PATCH 37/90] Add docstrings for MaubotMessageEvent methods --- maubot/matrix.py | 82 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/maubot/matrix.py b/maubot/matrix.py index 2387617..0f3e36b 100644 --- a/maubot/matrix.py +++ b/maubot/matrix.py @@ -86,6 +86,28 @@ class MaubotMessageEvent(MessageEvent): in_thread: bool | None = None, edits: EventID | MessageEvent | None = None, ) -> EventID: + """ + Respond to the message. + + Args: + content: The content to respond with. If this is a string, it will be passed to + :func:`parse_formatted` with the markdown and allow_html flags. + Otherwise, the content is used as-is + event_type: The type of event to send. + markdown: When content is a string, should it be parsed as markdown? + allow_html: When content is a string, should it allow raw HTML? + reply: Should the response be sent as a reply to this event? + in_thread: Should the response be sent in a thread with this event? + By default (``None``), the response will be in a thread if this event is in a + thread. If set to ``False``, the response will never be in a thread. If set to + ``True``, the response will always be in a thread, creating one with this event as + the root if necessary. + edits: An event ID or MessageEvent to edit. If set, the reply and in_thread parameters + are ignored, as edits can't change the reply or thread status. + + Returns: + The ID of the response event. + """ if isinstance(content, str): content = TextMessageEventContent(msgtype=MessageType.NOTICE, body=content) if allow_html or markdown: @@ -128,6 +150,26 @@ class MaubotMessageEvent(MessageEvent): allow_html: bool = False, in_thread: bool | None = None, ) -> Awaitable[EventID]: + """ + Reply to the message. The parameters are the same as :meth:`respond`, + but ``reply`` is always ``True`` and ``edits`` is not supported. + + Args: + content: The content to respond with. If this is a string, it will be passed to + :func:`parse_formatted` with the markdown and allow_html flags. + Otherwise, the content is used as-is + event_type: The type of event to send. + markdown: When content is a string, should it be parsed as markdown? + allow_html: When content is a string, should it allow raw HTML? + in_thread: Should the response be sent in a thread with this event? + By default (``None``), the response will be in a thread if this event is in a + thread. If set to ``False``, the response will never be in a thread. If set to + ``True``, the response will always be in a thread, creating one with this event as + the root if necessary. + + Returns: + The ID of the response event. + """ return self.respond( content, event_type, @@ -138,12 +180,37 @@ class MaubotMessageEvent(MessageEvent): ) def mark_read(self) -> Awaitable[None]: + """ + Mark this event as read. + """ return self.client.send_receipt(self.room_id, self.event_id, "m.read") def react(self, key: str) -> Awaitable[EventID]: + """ + React to this event with the given key. + + Args: + key: The key to react with. Often an unicode emoji. + + Returns: + The ID of the reaction event. + + Examples: + >>> evt: MaubotMessageEvent + >>> evt.react("🐈️") + """ return self.client.react(self.room_id, self.event_id, key) def redact(self, reason: str | None = None) -> Awaitable[EventID]: + """ + Redact this event. + + Args: + reason: Optionally, the reason for redacting the event. + + Returns: + The ID of the redaction event. + """ return self.client.redact(self.room_id, self.event_id, reason=reason) def edit( @@ -153,6 +220,21 @@ class MaubotMessageEvent(MessageEvent): markdown: bool = True, allow_html: bool = False, ) -> Awaitable[EventID]: + """ + Edit this event. Note that other clients will only render the edit if it was sent by the + same user who's doing the editing. + + Args: + content: The new content for the event. If this is a string, it will be passed to + :func:`parse_formatted` with the markdown and allow_html flags. + Otherwise, the content is used as-is. + event_type: The type of event to edit into. + markdown: When content is a string, should it be parsed as markdown? + allow_html: When content is a string, should it allow raw HTML? + + Returns: + The ID of the edit event. + """ return self.respond( content, event_type, markdown=markdown, edits=self, allow_html=allow_html ) From 719c9c5dd0a6c486eec83902b59122f740ba285b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 26 Feb 2023 16:15:23 +0200 Subject: [PATCH 38/90] Update changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20cb0bb..6690416 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +# unreleased + +* Added `in_thread` parameter to `evt.reply()` and `evt.respond()`. + * By default, responses will go to the thread if the command is in a thread. + * By setting the flag to `True` or `False`, the plugin can force the response + to either be or not be in a thread. +* Fixed `self.loader.meta` not being available to plugins in standalone mode. + # v0.4.0 (2023-01-29) * Dropped support for using a custom maubot API base path. From c9282b93f4bee4d13603dd7dab4c904c648ea53f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 6 Mar 2023 15:49:51 +0200 Subject: [PATCH 39/90] Fix serving static files. Fixes #206 --- maubot/server.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/maubot/server.py b/maubot/server.py index ac3132d..a8e72cf 100644 --- a/maubot/server.py +++ b/maubot/server.py @@ -127,6 +127,13 @@ class MaubotServer: ) self.app.router.add_get(ui_base, ui_base_redirect) + @staticmethod + def _static_data(data: bytes, mime: str) -> Callable[[web.Request], web.Response]: + def fn(_: web.Request) -> web.Response: + return web.Response(body=data, content_type=mime) + + return fn + def setup_static_root_files(self, directory: str, ui_base: str) -> None: files = { "asset-manifest.json": "application/json", @@ -136,9 +143,7 @@ class MaubotServer: for file, mime in files.items(): with open(f"{directory}/{file}", "rb") as stream: data = stream.read() - self.app.router.add_get( - f"{ui_base}/{file}", lambda _: web.Response(body=data, content_type=mime) - ) + self.app.router.add_get(f"{ui_base}/{file}", self._static_data(data, mime)) public_url = self.config["server.public_url"] public_url_path = "" From aaf32d8820a3a0e512359f031e19c03229707b2a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 13 Mar 2023 16:01:44 +0200 Subject: [PATCH 40/90] Update mautrix-python --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 50526b7..bebc1c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -mautrix>=0.19.4,<0.20 +mautrix>=0.19.6,<0.20 aiohttp>=3,<4 yarl>=1,<2 SQLAlchemy>=1,<1.4 From c2648be1e356dba5f76cada6b472349d6dee89ab Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 15 Mar 2023 20:00:31 +0200 Subject: [PATCH 41/90] Bump version to 0.4.1 --- CHANGELOG.md | 4 +++- maubot/__meta__.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6690416..014e4b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,12 @@ -# unreleased +# v0.4.1 (2023-03-15) * Added `in_thread` parameter to `evt.reply()` and `evt.respond()`. * By default, responses will go to the thread if the command is in a thread. * By setting the flag to `True` or `False`, the plugin can force the response to either be or not be in a thread. +* Fixed static files like the frontend app manifest not being served correctly. * Fixed `self.loader.meta` not being available to plugins in standalone mode. +* Updated to mautrix-python v0.19.6. # v0.4.0 (2023-01-29) diff --git a/maubot/__meta__.py b/maubot/__meta__.py index 6a9beea..3d26edf 100644 --- a/maubot/__meta__.py +++ b/maubot/__meta__.py @@ -1 +1 @@ -__version__ = "0.4.0" +__version__ = "0.4.1" From 28b0412c8cc15fc41a1edc81162916dd2526d34a Mon Sep 17 00:00:00 2001 From: Pierre GIRAUD Date: Fri, 24 Mar 2023 10:38:09 +0100 Subject: [PATCH 42/90] Adding a py.typed to avoid mypy errors when importing module This prevents errors like the following when importing maubot and running mypy: Skipping analyzing "maubot": module is installed, but missing library stubs or py.typed marker See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-library-stubs-or-py-typed-marker --- maubot/py.typed | 0 setup.py | 1 + 2 files changed, 1 insertion(+) create mode 100644 maubot/py.typed diff --git a/maubot/py.typed b/maubot/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py index 51687cc..2472222 100644 --- a/setup.py +++ b/setup.py @@ -69,6 +69,7 @@ setuptools.setup( "management/frontend/build/static/css/*", "management/frontend/build/static/js/*", "management/frontend/build/static/media/*", + "py.typed", ], "maubot.cli": ["res/*"], "maubot.standalone": ["example-config.yaml"], From 184d6b8eedf4710fc3ded28039264ed997e6336c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 7 Apr 2023 15:37:32 +0300 Subject: [PATCH 43/90] Update mautrix-python --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bebc1c1..550d116 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -mautrix>=0.19.6,<0.20 +mautrix>=0.19.8,<0.20 aiohttp>=3,<4 yarl>=1,<2 SQLAlchemy>=1,<1.4 From bf8ae9eb5aba7eaac70838e323228db910c27171 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Fri, 28 Apr 2023 15:31:00 -0600 Subject: [PATCH 44/90] standalone/Dockerfile: update to latest alpine Signed-off-by: Sumner Evans --- maubot/standalone/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maubot/standalone/Dockerfile b/maubot/standalone/Dockerfile index 8bf06f8..56e056d 100644 --- a/maubot/standalone/Dockerfile +++ b/maubot/standalone/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/alpine:3.15 +FROM docker.io/alpine:3.17 RUN apk add --no-cache \ python3 py3-pip py3-setuptools py3-wheel \ From 36829e7d0d886f4a8290c62f87855db703694127 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 26 Jun 2023 13:27:30 +0300 Subject: [PATCH 45/90] Update Docker image to Alpine 3.18 Closes #215 --- Dockerfile | 2 +- Dockerfile.ci | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 656a7c2..f445729 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM node:18 AS frontend-builder COPY ./maubot/management/frontend /frontend RUN cd /frontend && yarn --prod && yarn build -FROM alpine:3.17 +FROM alpine:3.18 RUN apk add --no-cache \ python3 py3-pip py3-setuptools py3-wheel \ diff --git a/Dockerfile.ci b/Dockerfile.ci index d3a7e32..82e2afc 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM alpine:3.17 +FROM alpine:3.18 RUN apk add --no-cache \ python3 py3-pip py3-setuptools py3-wheel \ From 96d4e434a8da1e7ffde9ebea0b19f03896947913 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 26 Jun 2023 13:35:00 +0300 Subject: [PATCH 46/90] Remove cchardet in dockerfile --- Dockerfile.ci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 82e2afc..72e4180 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -43,7 +43,7 @@ COPY optional-requirements.txt /opt/maubot/optional-requirements.txt WORKDIR /opt/maubot RUN apk add --virtual .build-deps python3-dev build-base git \ && pip3 install -r requirements.txt -r optional-requirements.txt \ - dateparser langdetect python-gitlab pyquery cchardet semver tzlocal cssselect \ + dateparser langdetect python-gitlab pyquery semver tzlocal cssselect \ && apk del .build-deps # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies From a4253eceb24c65abba58bd3ada7d3d668f62b79c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 26 Jun 2023 15:26:53 +0300 Subject: [PATCH 47/90] Move plugin list to separate website Closes #214 Closes #212 Closes #208 Closes #201 Closes #199 --- README.md | 68 +++---------------------------------------------------- 1 file changed, 3 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 6dbce90..a2eda36 100644 --- a/README.md +++ b/README.md @@ -22,70 +22,8 @@ All setup and usage instructions are located on Matrix room: [#maubot:maunium.net](https://matrix.to/#/#maubot:maunium.net) ## Plugins -Open a pull request or join the Matrix room linked above to get your plugin listed here. +A list of plugins can be found at [plugins.maubot.xyz](https://plugins.maubot.xyz/). + +To add your plugin to the list, send a pull request to . The plugin wishlist lives at . - -### Official plugins -* [sed](https://github.com/maubot/sed) - A bot to do sed-like replacements. -* [factorial](https://github.com/maubot/factorial) - A bot to calculate unexpected factorials. -* [media](https://github.com/maubot/media) - A bot that replies with the MXC URI of images you send it. -* [dice](https://github.com/maubot/dice) - A combined dice rolling and calculator bot. -* [karma](https://github.com/maubot/karma) - A user karma tracker bot. -* [xkcd](https://github.com/maubot/xkcd) - A bot to view xkcd comics. -* [echo](https://github.com/maubot/echo) - A bot that echoes pings and other stuff. -* [rss](https://github.com/maubot/rss) - A bot that posts RSS feed updates to Matrix. -* [reminder](https://github.com/maubot/reminder) - A bot to remind you about things. -* [translate](https://github.com/maubot/translate) - A bot to translate words. -* [reactbot](https://github.com/maubot/reactbot) - A bot that responds to messages that match predefined rules. -* [exec](https://github.com/maubot/exec) - A bot that executes code. -* [commitstrip](https://github.com/maubot/commitstrip) - A bot to view CommitStrips. -* [supportportal](https://github.com/maubot/supportportal) - A bot to manage customer support on Matrix. -* †[gitlab](https://github.com/maubot/gitlab) - A GitLab client and webhook receiver. -* [github](https://github.com/maubot/github) - A GitHub client and webhook receiver. -* [tex](https://github.com/maubot/tex) - A bot that renders LaTeX. -* [altalias](https://github.com/maubot/altalias) - A bot that lets users publish alternate aliases in rooms. -* [satwcomic](https://github.com/maubot/satwcomic) - A bot to view SatWComics. -* [songwhip](https://github.com/maubot/songwhip) - A bot to post Songwhip links. -* [manhole](https://github.com/maubot/manhole) - A plugin that lets you access a Python shell inside maubot. - -### 3rd party plugins -* [subreddit linkifier](https://github.com/TomCasavant/RedditMaubot) - A bot that condescendingly corrects a user when they enter an r/subreddit without providing a link to that subreddit -* [giphy](https://github.com/TomCasavant/GiphyMaubot) - A bot that generates a gif (from giphy) given search terms -* [trump](https://github.com/jeffcasavant/MaubotTrumpTweet) - A bot that generates a Trump tweet with the given content -* [poll](https://github.com/TomCasavant/PollMaubot) - A bot that will create a simple poll for users in a room -* [urban](https://github.com/dvdgsng/UrbanMaubot) - A bot that fetches definitions from [Urban Dictionary](https://www.urbandictionary.com/). -* [twilio](https://github.com/jeffcasavant/MaubotTwilio) - Maubot-based SMS bridge -* [tmdb](https://codeberg.org/lomion/tmdb-bot) - A bot that posts information about movies fetched from TheMovieDB.org. -* [invite](https://github.com/williamkray/maubot-invite) - A bot to generate invitation tokens from [matrix-registration](https://github.com/ZerataX/matrix-registration) -* [wolframalpha](https://github.com/ggogel/WolframAlphaMaubot) - A bot that allows requesting information from [WolframAlpha](https://www.wolframalpha.com/). -* †[pingcheck](https://edugit.org/nik/maubot-pingcheck) - A bot to ping the echo bot and send rtt to Icinga passive check -* [ticker](https://github.com/williamkray/maubot-ticker) - A bot to return financial data about a stock or cryptocurrency. -* [weather](https://github.com/kellya/maubot-weather) - A bot to get the weather from wttr.in and return a single line of text for the location specified -* †[youtube previewer](https://github.com/ggogel/YoutubePreviewMaubot) - A bot that responds to a YouTube link with the video title and thumbnail. -* †[reddit previewer](https://github.com/ggogel/RedditPreviewMaubot) - A bot that responds to a link of a reddit post with the sub name and title. If available, uploads the image or video. -* [pocket](https://github.com/jaywink/maubot-pocket) - A bot integrating with Pocket to fetch articles and archive them. -* [alternatingcaps](https://github.com/rom4nik/maubot-alternatingcaps) - A bot repeating last message using aLtErNaTiNg cApS. -* [metric](https://github.com/edwardsdean/maubot_metric_bot) - A bot that will reply to a message that contains imperial units and replace them with metric units. -* [urlpreview](https://github.com/coffeebank/coffee-maubot/tree/master/urlpreview) - A bot that responds to links with a link preview embed, using Matrix API to fetch meta tags. -* [autoreply](https://github.com/babolivier/maubot-autoreply) - A bot that sends automated replies when you're away, and shows you a summary of missed messages when you come back. -* [alertbot](https://github.com/moan0s/alertbot) - A bot that recives monitoring alerts via alertmanager and forwards them to a matrix room. -* [hasswebhookbot](https://github.com/v411e/hasswebhookbot) - A bot receiving webhooks from [Home Assistant](https://github.com/home-assistant). -* [ovgumensabot](https://github.com/v411e/ovgumensabot) - A bot that automatically sends meals from OvGU canteen every day. -* †[token](https://github.com/yoxcu/maubot-token) - A maubot to create and manage your synapse user registration tokens. -* [redactbot](https://gitlab.com/sspaeth/redactbot) - A bot that immediately redacts any posted file (except for whitelisted types). -* [join](https://github.com/williamkray/maubot-join) - A plugin that restricts who can convince your bot to join new rooms to certain users. -* [create-room](https://github.com/williamkray/maubot-createroom) - A plugin that creates new rooms and automatically - sets them to be part of a private Matrix Space. -* [welcome](https://github.com/williamkray/maubot-welcome) - A plugin that greets new people with a configurable message when they join a room. -* [activity tracker](https://github.com/williamkray/maubot-kickbot) - A plugin that minimally tracks user activity - within a space. Useful for kicking inactive users from a private community. -* [random subreddit post](https://github.com/williamkray/maubot-reddit) - A plugin that returns a random post from a - given subreddit. - -† Uses a synchronous library which can block the whole maubot process (e.g. requests instead of aiohttp) - -### Deprecated/unmaintained plugins -* [jesaribot](https://github.com/maubot/jesaribot) - A simple bot that replies with an image when you say "jesari". - * Superseded by reactbot -* [gitea](https://github.com/saces/maugitea) - A Gitea client and webhook receiver. From ed046bcbfe9a9c8daedc89bc04debfbc9ed6b515 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 30 Aug 2023 17:05:12 +0300 Subject: [PATCH 48/90] Log errors when calling whoami for new clients --- maubot/management/api/client.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/maubot/management/api/client.py b/maubot/management/api/client.py index d95286b..2a8964c 100644 --- a/maubot/management/api/client.py +++ b/maubot/management/api/client.py @@ -16,6 +16,7 @@ from __future__ import annotations from json import JSONDecodeError +import logging from aiohttp import web @@ -27,6 +28,8 @@ from ...client import Client from .base import routes from .responses import resp +log = logging.getLogger("maubot.server.client") + @routes.get("/clients") async def get_clients(_: web.Request) -> web.Response: @@ -54,11 +57,13 @@ async def _create_client(user_id: UserID | None, data: dict) -> web.Response: ) try: whoami = await new_client.whoami() - except MatrixInvalidToken: + except MatrixInvalidToken as e: return resp.bad_client_access_token except MatrixRequestError: + log.warning(f"Failed to get whoami from {homeserver} for new client", exc_info=True) return resp.bad_client_access_details except MatrixConnectionError: + log.warning(f"Failed to connect to {homeserver} for new client", exc_info=True) return resp.bad_client_connection_details if user_id is None: existing_client = await Client.get(whoami.user_id) @@ -90,8 +95,12 @@ async def _update_client(client: Client, data: dict, is_login: bool = False) -> except MatrixInvalidToken: return resp.bad_client_access_token except MatrixRequestError: + log.warning( + f"Failed to get whoami from homeserver to update client details", exc_info=True + ) return resp.bad_client_access_details except MatrixConnectionError: + log.warning(f"Failed to connect to homeserver to update client details", exc_info=True) return resp.bad_client_connection_details except ValueError as e: str_err = str(e) From 61f154876d99ad95107d639d255fb7edd2e6cd76 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 29 Jun 2023 15:44:46 +0300 Subject: [PATCH 49/90] Allow multiple event types in the same handler --- maubot/handlers/command.py | 6 +++--- maubot/handlers/event.py | 7 +++++-- maubot/plugin_base.py | 5 +++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/maubot/handlers/command.py b/maubot/handlers/command.py index 1d9cdfb..27e6547 100644 --- a/maubot/handlers/command.py +++ b/maubot/handlers/command.py @@ -72,7 +72,7 @@ class CommandHandler: self.__mb_must_consume_args__: bool = True self.__mb_arg_fallthrough__: bool = True self.__mb_event_handler__: bool = True - self.__mb_event_type__: EventType = EventType.ROOM_MESSAGE + self.__mb_event_types__: set[EventType] = {EventType.ROOM_MESSAGE} self.__mb_msgtypes__: Iterable[MessageType] = (MessageType.TEXT,) self.__bound_copies__: Dict[Any, CommandHandler] = {} self.__bound_instance__: Any = None @@ -95,7 +95,7 @@ class CommandHandler: "must_consume_args", "arg_fallthrough", "event_handler", - "event_type", + "event_types", "msgtypes", ] for key in keys: @@ -315,7 +315,7 @@ def new( func.__mb_require_subcommand__ = require_subcommand func.__mb_arg_fallthrough__ = arg_fallthrough func.__mb_must_consume_args__ = must_consume_args - func.__mb_event_type__ = event_type + func.__mb_event_types__ = {event_type} if msgtypes: func.__mb_msgtypes__ = msgtypes return func diff --git a/maubot/handlers/event.py b/maubot/handlers/event.py index a9f8ac8..9be89b1 100644 --- a/maubot/handlers/event.py +++ b/maubot/handlers/event.py @@ -27,9 +27,12 @@ def on(var: EventType | InternalEventType | EventHandler) -> EventHandlerDecorat def decorator(func: EventHandler) -> EventHandler: func.__mb_event_handler__ = True if isinstance(var, (EventType, InternalEventType)): - func.__mb_event_type__ = var + if hasattr(func, "__mb_event_types__"): + func.__mb_event_types__.add(var) + else: + func.__mb_event_types__ = {var} else: - func.__mb_event_type__ = EventType.ALL + func.__mb_event_types__ = {EventType.ALL} return func diff --git a/maubot/plugin_base.py b/maubot/plugin_base.py index 396a7b6..d7b6bbd 100644 --- a/maubot/plugin_base.py +++ b/maubot/plugin_base.py @@ -76,8 +76,9 @@ class Plugin(ABC): val = getattr(obj, key) try: if val.__mb_event_handler__: - self._handlers_at_startup.append((val, val.__mb_event_type__)) - self.client.add_event_handler(val.__mb_event_type__, val) + for event_type in val.__mb_event_types__: + self._handlers_at_startup.append((val, event_type)) + self.client.add_event_handler(event_type, val) except AttributeError: pass try: From 8f40a0b292f1f4f7c1a28d54b0530a37add4de5a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Sep 2023 21:39:28 +0300 Subject: [PATCH 50/90] Update mautrix-python --- docker/run.sh | 2 +- maubot/config.py | 6 +++++- maubot/example-config.yaml | 4 ++-- maubot/instance.py | 2 +- maubot/standalone/example-config.yaml | 2 +- requirements.txt | 2 +- setup.py | 3 +-- 7 files changed, 12 insertions(+), 9 deletions(-) diff --git a/docker/run.sh b/docker/run.sh index ff9b54c..5447e27 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -13,7 +13,7 @@ function fixdefault { function fixconfig { # Change relative default paths to absolute paths in /data - fixdefault '.database' 'sqlite:///maubot.db' 'sqlite:////data/maubot.db' + fixdefault '.database' 'sqlite:maubot.db' 'sqlite:/data/maubot.db' fixdefault '.plugin_directories.upload' './plugins' '/data/plugins' fixdefault '.plugin_directories.load[0]' './plugins' '/data/plugins' fixdefault '.plugin_directories.trash' './trash' '/data/trash' diff --git a/maubot/config.py b/maubot/config.py index c5368cd..b8e42de 100644 --- a/maubot/config.py +++ b/maubot/config.py @@ -32,7 +32,11 @@ class Config(BaseFileConfig): def do_update(self, helper: ConfigUpdateHelper) -> None: base = helper.base copy = helper.copy - copy("database") + + if "database" in self and self["database"].startswith("sqlite:///"): + helper.base["database"] = self["database"].replace("sqlite:///", "sqlite:") + else: + copy("database") copy("database_opts") if isinstance(self["crypto_database"], dict): if self["crypto_database.type"] == "postgres": diff --git a/maubot/example-config.yaml b/maubot/example-config.yaml index 0181846..a16ea19 100644 --- a/maubot/example-config.yaml +++ b/maubot/example-config.yaml @@ -1,9 +1,9 @@ # The full URI to the database. SQLite and Postgres are fully supported. # Other DBMSes supported by SQLAlchemy may or may not work. # Format examples: -# SQLite: sqlite:///filename.db +# SQLite: sqlite:filename.db # Postgres: postgresql://username:password@hostname/dbname -database: sqlite:///maubot.db +database: sqlite:maubot.db # Separate database URL for the crypto database. "default" means use the same database as above. crypto_database: default diff --git a/maubot/instance.py b/maubot/instance.py index c63e5ab..2905d12 100644 --- a/maubot/instance.py +++ b/maubot/instance.py @@ -317,7 +317,7 @@ class PluginInstance(DBInstance): ) else: self.inst_db = Database.create( - f"sqlite:///{self._sqlite_db_path}", + f"sqlite:{self._sqlite_db_path}", upgrade_table=upgrade_table, log=instance_db_log, ) diff --git a/maubot/standalone/example-config.yaml b/maubot/standalone/example-config.yaml index 1884b78..457a5ca 100644 --- a/maubot/standalone/example-config.yaml +++ b/maubot/standalone/example-config.yaml @@ -35,7 +35,7 @@ server: # The database for the plugin. Used for plugin data, the sync token and e2ee data (if enabled). # SQLite and Postgres are supported. -database: sqlite:///bot.db +database: sqlite:bot.db # Additional arguments for asyncpg.create_pool() or sqlite3.connect() # https://magicstack.github.io/asyncpg/current/api/index.html#asyncpg.pool.create_pool diff --git a/requirements.txt b/requirements.txt index 550d116..e5743ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -mautrix>=0.19.8,<0.20 +mautrix>=0.20.1,<0.21 aiohttp>=3,<4 yarl>=1,<2 SQLAlchemy>=1,<1.4 diff --git a/setup.py b/setup.py index 2472222..24f9e00 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setuptools.setup( install_requires=install_requires, extras_require=extras_require, - python_requires="~=3.8", + python_requires="~=3.9", classifiers=[ "Development Status :: 4 - Beta", @@ -50,7 +50,6 @@ setuptools.setup( "Framework :: AsyncIO", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", From 92736baefdac0298cf4c96bc57a5247b210eabd6 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Sep 2023 21:35:14 +0300 Subject: [PATCH 51/90] Add appservice option to standalone mode --- maubot/server.py | 1 + maubot/standalone/__main__.py | 61 +++++++++++++++++++++++++-- maubot/standalone/config.py | 4 ++ maubot/standalone/example-config.yaml | 9 +++- 4 files changed, 71 insertions(+), 4 deletions(-) diff --git a/maubot/server.py b/maubot/server.py index a8e72cf..097fe5b 100644 --- a/maubot/server.py +++ b/maubot/server.py @@ -15,6 +15,7 @@ # along with this program. If not, see . from __future__ import annotations +from typing import Callable import asyncio import json import logging diff --git a/maubot/standalone/__main__.py b/maubot/standalone/__main__.py index 87adc35..6d3150d 100644 --- a/maubot/standalone/__main__.py +++ b/maubot/standalone/__main__.py @@ -30,7 +30,11 @@ from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap from yarl import URL +from mautrix.appservice import AppServiceServerMixin +from mautrix.client import SyncStream from mautrix.types import ( + BaseMessageEventContentFuncs, + Event, EventType, Filter, Membership, @@ -113,6 +117,9 @@ if "/" in meta.main_class: else: module = meta.modules[0] main_class = meta.main_class + +if args.meta != "maubot.yaml" and os.path.dirname(args.meta) != "": + sys.path.append(os.path.dirname(args.meta)) bot_module = importlib.import_module(module) plugin: type[Plugin] = getattr(bot_module, main_class) loader = FileSystemLoader(os.path.dirname(args.meta), meta) @@ -131,6 +138,7 @@ user_id = config["user.credentials.id"] device_id = config["user.credentials.device_id"] homeserver = config["user.credentials.homeserver"] access_token = config["user.credentials.access_token"] +appservice_listener = config["user.appservice"] crypto_store = state_store = None if device_id and not OlmMachine: @@ -188,6 +196,10 @@ if meta.webapp: resource = PrefixResource(web_base_path) resource.add_route(hdrs.METH_ANY, _handle_plugin_request) web_app.router.register_resource(resource) +elif appservice_listener: + web_app = web.Application() + web_runner = web.AppRunner(web_app, access_log_class=AccessLogger) + public_url = plugin_webapp = None else: web_app = web_runner = public_url = plugin_webapp = None @@ -195,6 +207,31 @@ loop = asyncio.get_event_loop() client: MaubotMatrixClient | None = None bot: Plugin | None = None +appservice: AppServiceServerMixin | None = None + + +if appservice_listener: + assert web_app is not None, "web_app is always set when appservice_listener is set" + appservice = AppServiceServerMixin( + ephemeral_events=True, + encryption_events=True, + log=logging.getLogger("maubot.appservice"), + hs_token=config["user.hs_token"], + ) + appservice.register_routes(web_app) + + @appservice.matrix_event_handler + async def handle_appservice_event(evt: Event) -> None: + if isinstance(evt.content, BaseMessageEventContentFuncs): + evt.content.trim_reply_fallback() + fake_sync_stream = SyncStream.JOINED_ROOM + if evt.type.is_ephemeral: + fake_sync_stream |= SyncStream.EPHEMERAL + else: + fake_sync_stream |= SyncStream.TIMELINE + setattr(evt, "source", fake_sync_stream) + tasks = client.dispatch_manual_event(evt.type, evt, include_global_handlers=True) + await asyncio.gather(*tasks) async def main(): @@ -217,6 +254,8 @@ async def main(): state_store=state_store, device_id=device_id, ) + if appservice: + client.api.as_user_id = user_id client.ignore_first_sync = config["user.ignore_first_sync"] client.ignore_initial_sync = config["user.ignore_initial_sync"] if crypto_store: @@ -225,6 +264,11 @@ async def main(): await crypto_store.open() client.crypto = OlmMachine(client, crypto_store, state_store) + if appservice: + appservice.otk_handler = client.crypto.handle_as_otk_counts + appservice.device_list_handler = client.crypto.handle_as_device_lists + appservice.to_device_handler = client.crypto.handle_as_to_device_event + client.api.as_device_id = device_id crypto_device_id = await crypto_store.get_device_id() if crypto_device_id and crypto_device_id != device_id: log.fatal( @@ -272,6 +316,8 @@ async def main(): ) await nb.put_filter_id(filter_id) _ = client.start(nb.filter_id) + elif appservice_listener and crypto_store and not client.crypto.account.shared: + await client.crypto.share_keys() if config["user.autojoin"]: log.debug("Autojoin is enabled") @@ -334,9 +380,14 @@ async def stop(suppress_stop_error: bool = False) -> None: except Exception: if not suppress_stop_error: log.exception("Error stopping bot") - if web_runner: - await web_runner.shutdown() - await web_runner.cleanup() + if web_runner and web_runner.server: + try: + await web_runner.shutdown() + await web_runner.cleanup() + except RuntimeError: + if not suppress_stop_error: + await db.stop() + raise await db.stop() @@ -347,6 +398,10 @@ signal.signal(signal.SIGTERM, signal.default_int_handler) try: log.info("Starting plugin") loop.run_until_complete(main()) +except SystemExit: + loop.run_until_complete(stop(suppress_stop_error=True)) + loop.close() + raise except (Exception, KeyboardInterrupt) as e: if isinstance(e, KeyboardInterrupt): log.info("Startup interrupted, stopping") diff --git a/maubot/standalone/config.py b/maubot/standalone/config.py index ce17310..f2c90a0 100644 --- a/maubot/standalone/config.py +++ b/maubot/standalone/config.py @@ -33,9 +33,13 @@ class Config(BaseFileConfig): copy("user.credentials.access_token") copy("user.credentials.device_id") copy("user.sync") + copy("user.appservice") + copy("user.hs_token") copy("user.autojoin") copy("user.displayname") copy("user.avatar_url") + copy("user.ignore_initial_sync") + copy("user.ignore_first_sync") if "server" in base: copy("server.hostname") copy("server.port") diff --git a/maubot/standalone/example-config.yaml b/maubot/standalone/example-config.yaml index 457a5ca..8ee4e43 100644 --- a/maubot/standalone/example-config.yaml +++ b/maubot/standalone/example-config.yaml @@ -5,9 +5,15 @@ user: homeserver: https://example.com access_token: foo # If you want to enable encryption, set the device ID corresponding to the access token here. + # When using an appservice, you should use appservice login manually to generate a device ID and access token. device_id: null # Enable /sync? This is not needed for purely unencrypted webhook-based bots, but is necessary in most other cases. sync: true + # Receive appservice transactions? This will add a /_matrix/app/v1/transactions endpoint on + # the HTTP server configured below. The base_path will not be applied for the /transactions path. + appservice: false + # When appservice mode is enabled, the hs_token for the appservice. + hs_token: null # Automatically accept invites? autojoin: false # The displayname and avatar URL to set for the bot on startup. @@ -21,7 +27,8 @@ user: # if you want the bot to handle messages that were sent while the bot was down. ignore_first_sync: true -# Web server settings. These will only take effect if the plugin requests it using `webapp: true` in the meta file. +# Web server settings. These will only take effect if the plugin requests it using `webapp: true` in the meta file, +# or if user -> appservice is set to true. server: # The IP and port to listen to. hostname: 0.0.0.0 From dd69c337d2bbcf05b3b602d01e2e40b766dbf2c3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 9 Sep 2023 15:47:53 +0300 Subject: [PATCH 52/90] Update standalone docker image and mautrix-python --- maubot/standalone/Dockerfile | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/maubot/standalone/Dockerfile b/maubot/standalone/Dockerfile index 56e056d..2db8426 100644 --- a/maubot/standalone/Dockerfile +++ b/maubot/standalone/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/alpine:3.17 +FROM docker.io/alpine:3.18 RUN apk add --no-cache \ python3 py3-pip py3-setuptools py3-wheel \ diff --git a/requirements.txt b/requirements.txt index e5743ae..b85decc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -mautrix>=0.20.1,<0.21 +mautrix>=0.20.2,<0.21 aiohttp>=3,<4 yarl>=1,<2 SQLAlchemy>=1,<1.4 From b619d3ad566b4140b3a9edbd65afabade5615978 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 19 Sep 2023 13:23:04 -0400 Subject: [PATCH 53/90] Update Pillow in docker image --- Dockerfile | 5 +++-- requirements.txt | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index f445729..f4bd0df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,12 +34,13 @@ RUN apk add --no-cache \ py3-unpaddedbase64 \ py3-future \ # plugin deps - py3-pillow \ + #py3-pillow \ py3-magic \ py3-feedparser \ py3-dateutil \ py3-lxml \ - py3-semver + py3-semver \ + && apk add --no-cache py3-pillow --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community # TODO remove pillow, magic, feedparser, lxml, gitlab and semver when maubot supports installing dependencies COPY requirements.txt /opt/maubot/requirements.txt diff --git a/requirements.txt b/requirements.txt index b85decc..4134190 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ mautrix>=0.20.2,<0.21 aiohttp>=3,<4 yarl>=1,<2 SQLAlchemy>=1,<1.4 -asyncpg>=0.20,<0.28 +asyncpg>=0.20,<0.29 aiosqlite>=0.16,<0.19 commonmark>=0.9,<1 ruamel.yaml>=0.15.35,<0.18 From 923125f7930a100bd9176e5959be5446c7e429d7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 19 Sep 2023 13:28:08 -0400 Subject: [PATCH 54/90] Update pillow in CI dockerfile too --- Dockerfile.ci | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 72e4180..7a957f8 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -30,10 +30,11 @@ RUN apk add --no-cache \ py3-unpaddedbase64 \ py3-future \ # plugin deps - py3-pillow \ + #py3-pillow \ py3-magic \ py3-feedparser \ - py3-lxml + py3-lxml \ + && apk add --no-cache py3-pillow --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community # py3-gitlab # py3-semver # TODO remove pillow, magic, feedparser, lxml, gitlab and semver when maubot supports installing dependencies From 75879cfb9370aade6fa0e84e1dde47222625139a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 30 Sep 2023 14:12:33 +0300 Subject: [PATCH 55/90] Bump version to 0.4.2 --- CHANGELOG.md | 8 ++++++++ maubot/__meta__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 014e4b0..2a3e691 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +# v0.4.2 (2023-09-20) + +* Updated Pillow to 10.0.1. +* Updated Docker image to Alpine 3.18. +* Added logging for errors for /whoami errors when adding new bot accounts. +* Added support for using appservice tokens (including appservice encryption) + in standalone mode. + # v0.4.1 (2023-03-15) * Added `in_thread` parameter to `evt.reply()` and `evt.respond()`. diff --git a/maubot/__meta__.py b/maubot/__meta__.py index 3d26edf..df12433 100644 --- a/maubot/__meta__.py +++ b/maubot/__meta__.py @@ -1 +1 @@ -__version__ = "0.4.1" +__version__ = "0.4.2" From 202c2836b2d4faea34fdd94b179baa77c0374992 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Mon, 6 Nov 2023 15:32:52 +0100 Subject: [PATCH 56/90] Add a testing framework MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This changeset contains a set of Pytest fixtures and a mocked bot class to ease the writing of Maubot plugin unit tests. Signed-off-by: Aurélien Bompard --- maubot/testing/__init__.py | 17 +++++ maubot/testing/bot.py | 100 +++++++++++++++++++++++++++ maubot/testing/fixtures.py | 135 +++++++++++++++++++++++++++++++++++++ optional-requirements.txt | 4 ++ setup.py | 2 + 5 files changed, 258 insertions(+) create mode 100644 maubot/testing/__init__.py create mode 100644 maubot/testing/bot.py create mode 100644 maubot/testing/fixtures.py diff --git a/maubot/testing/__init__.py b/maubot/testing/__init__.py new file mode 100644 index 0000000..1fcdfc0 --- /dev/null +++ b/maubot/testing/__init__.py @@ -0,0 +1,17 @@ +# maubot - A plugin-based Matrix bot system. +# Copyright (C) 2023 Aurélien Bompard +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from .bot import TestBot, make_message # noqa: F401 +from .fixtures import * # noqa: F401,F403 diff --git a/maubot/testing/bot.py b/maubot/testing/bot.py new file mode 100644 index 0000000..0519016 --- /dev/null +++ b/maubot/testing/bot.py @@ -0,0 +1,100 @@ +# maubot - A plugin-based Matrix bot system. +# Copyright (C) 2023 Aurélien Bompard +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +import asyncio +import time + +from attr import dataclass + +from maubot.matrix import MaubotMatrixClient, MaubotMessageEvent +from mautrix.api import HTTPAPI +from mautrix.types import ( + EventContent, + EventType, + MessageEvent, + MessageType, + RoomID, + TextMessageEventContent, +) + + +@dataclass +class MatrixEvent: + room_id: RoomID + event_type: EventType + content: EventContent + kwargs: dict + + +class TestBot: + """A mocked bot used for testing purposes. + + Send messages to the mock Matrix server with the ``send()`` method. + Look into the ``responded`` list to get what server has replied. + """ + + def __init__(self, mxid="@botname:example.com", mxurl="http://matrix.example.com"): + api = HTTPAPI(base_url=mxurl) + self.client = MaubotMatrixClient(api=api) + self.responded = [] + self.client.mxid = mxid + self.client.send_message_event = self._mock_send_message_event + + async def _mock_send_message_event(self, room_id, event_type, content, txn_id=None, **kwargs): + self.responded.append( + MatrixEvent(room_id=room_id, event_type=event_type, content=content, kwargs=kwargs) + ) + + async def dispatch(self, event_type: EventType, event): + tasks = self.client.dispatch_manual_event(event_type, event, force_synchronous=True) + return await asyncio.gather(*tasks) + + async def send( + self, + content, + html=None, + room_id="testroom", + msg_type=MessageType.TEXT, + sender="@dummy:example.com", + timestamp=None, + ): + event = make_message( + content, + html=html, + room_id=room_id, + msg_type=msg_type, + sender=sender, + timestamp=timestamp, + ) + await self.dispatch(EventType.ROOM_MESSAGE, MaubotMessageEvent(event, self.client)) + + +def make_message( + content, + html=None, + room_id="testroom", + msg_type=MessageType.TEXT, + sender="@dummy:example.com", + timestamp=None, +): + """Make a Matrix message event.""" + return MessageEvent( + type=EventType.ROOM_MESSAGE, + room_id=room_id, + event_id="test", + sender=sender, + timestamp=timestamp or int(time.time() * 1000), + content=TextMessageEventContent(msgtype=msg_type, body=content, formatted_body=html), + ) diff --git a/maubot/testing/fixtures.py b/maubot/testing/fixtures.py new file mode 100644 index 0000000..e975782 --- /dev/null +++ b/maubot/testing/fixtures.py @@ -0,0 +1,135 @@ +# maubot - A plugin-based Matrix bot system. +# Copyright (C) 2023 Aurélien Bompard +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from pathlib import Path +import asyncio +import logging + +from ruamel.yaml import YAML +import aiohttp +import pytest +import pytest_asyncio + +from maubot import Plugin +from maubot.loader import PluginMeta +from maubot.standalone.loader import FileSystemLoader +from mautrix.util.async_db import Database +from mautrix.util.config import BaseProxyConfig, RecursiveDict +from mautrix.util.logging import TraceLogger + +from .bot import TestBot + + +@pytest_asyncio.fixture +async def maubot_test_bot(): + return TestBot() + + +@pytest.fixture +def maubot_upgrade_table(): + return None + + +@pytest.fixture +def maubot_plugin_path(): + return Path(".") + + +@pytest.fixture +def maubot_plugin_meta(maubot_plugin_path): + yaml = YAML() + with open(maubot_plugin_path.joinpath("maubot.yaml")) as fh: + plugin_meta = PluginMeta.deserialize(yaml.load(fh.read())) + return plugin_meta + + +@pytest_asyncio.fixture +async def maubot_plugin_db(tmp_path, maubot_plugin_meta, maubot_upgrade_table): + if not maubot_plugin_meta.get("database", False): + return + db_path = tmp_path.joinpath("maubot-tests.db").as_posix() + db = Database.create( + f"sqlite:{db_path}", + upgrade_table=maubot_upgrade_table, + log=logging.getLogger("db"), + ) + await db.start() + yield db + await db.stop() + + +@pytest.fixture +def maubot_plugin_class(): + return Plugin + + +@pytest.fixture +def maubot_plugin_config_class(): + return BaseProxyConfig + + +@pytest.fixture +def maubot_plugin_config_dict(): + return {} + + +@pytest.fixture +def maubot_plugin_config_overrides(): + return {} + + +@pytest.fixture +def maubot_plugin_config( + maubot_plugin_path, + maubot_plugin_config_class, + maubot_plugin_config_dict, + maubot_plugin_config_overrides, +): + yaml = YAML() + with open(maubot_plugin_path.joinpath("base-config.yaml")) as fh: + base_config = RecursiveDict(yaml.load(fh)) + maubot_plugin_config_dict.update(maubot_plugin_config_overrides) + return maubot_plugin_config_class( + load=lambda: maubot_plugin_config_dict, + load_base=lambda: base_config, + save=lambda c: None, + ) + + +@pytest_asyncio.fixture +async def maubot_plugin( + maubot_test_bot, + maubot_plugin_db, + maubot_plugin_class, + maubot_plugin_path, + maubot_plugin_config, + maubot_plugin_meta, +): + loader = FileSystemLoader(maubot_plugin_path, maubot_plugin_meta) + async with aiohttp.ClientSession() as http: + instance = maubot_plugin_class( + client=maubot_test_bot.client, + loop=asyncio.get_running_loop(), + http=http, + instance_id="tests", + log=TraceLogger("test"), + config=maubot_plugin_config, + database=maubot_plugin_db, + webapp=None, + webapp_url=None, + loader=loader, + ) + await instance.internal_start() + yield instance diff --git a/optional-requirements.txt b/optional-requirements.txt index 6d87db3..0e45b97 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -5,3 +5,7 @@ python-olm>=3,<4 pycryptodome>=3,<4 unpaddedbase64>=1,<3 + +#/testing +pytest +pytest-asyncio diff --git a/setup.py b/setup.py index 24f9e00..79a0c6c 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,8 @@ setuptools.setup( entry_points=""" [console_scripts] mbc=maubot.cli:app + [pytest11] + maubot=maubot.testing """, data_files=[ (".", ["maubot/example-config.yaml"]), From 4184280d4e13728b2aeb11867af79f0c6a98f678 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 7 Mar 2024 16:14:06 +0200 Subject: [PATCH 57/90] Add basic scheduler for plugins --- maubot/plugin_base.py | 5 ++ maubot/scheduler.py | 159 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 maubot/scheduler.py diff --git a/maubot/plugin_base.py b/maubot/plugin_base.py index d7b6bbd..5e967dc 100644 --- a/maubot/plugin_base.py +++ b/maubot/plugin_base.py @@ -27,6 +27,8 @@ from mautrix.util.async_db import Database, UpgradeTable from mautrix.util.config import BaseProxyConfig from mautrix.util.logging import TraceLogger +from .scheduler import BasicScheduler + if TYPE_CHECKING: from .client import MaubotMatrixClient from .loader import BasePluginLoader @@ -40,6 +42,7 @@ class Plugin(ABC): log: TraceLogger loop: AbstractEventLoop loader: BasePluginLoader + sched: BasicScheduler config: BaseProxyConfig | None database: Engine | Database | None webapp: PluginWebApp | None @@ -58,6 +61,7 @@ class Plugin(ABC): webapp_url: str | None, loader: BasePluginLoader, ) -> None: + self.sched = BasicScheduler(log=log.getChild("scheduler")) self.client = client self.loop = loop self.http = http @@ -117,6 +121,7 @@ class Plugin(ABC): self.client.remove_event_handler(event_type, func) if self.webapp is not None: self.webapp.clear() + self.sched.stop() await self.stop() async def stop(self) -> None: diff --git a/maubot/scheduler.py b/maubot/scheduler.py new file mode 100644 index 0000000..0cb39ed --- /dev/null +++ b/maubot/scheduler.py @@ -0,0 +1,159 @@ +# maubot - A plugin-based Matrix bot system. +# Copyright (C) 2024 Tulir Asokan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from __future__ import annotations + +from typing import Awaitable, Callable +import asyncio +import logging + + +class BasicScheduler: + background_loop: asyncio.Task | None + tasks: set[asyncio.Task] + log: logging.Logger + + def __init__(self, log: logging.Logger) -> None: + self.log = log + self.tasks = set() + + def _find_caller(self) -> str: + try: + file_name, line_number, function_name, _ = self.log.findCaller() + return f"{function_name} at {file_name}:{line_number}" + except ValueError: + return "unknown function" + + def run_periodically( + self, + period: float | int, + func: Callable[[], Awaitable], + run_task_in_background: bool = False, + catch_errors: bool = True, + ) -> asyncio.Task: + """ + Run a function periodically in the background. + + Args: + period: The period in seconds between each call to the function. + func: The function to run. No parameters will be provided, + use :meth:`functools.partial` if you need to pass parameters. + run_task_in_background: If ``True``, the function will be run in a background task. + If ``False`` (the default), the loop will wait for the task to return before + sleeping for the next period. + catch_errors: Whether the scheduler should catch and log any errors. + If ``False``, errors will be raised, and the caller must await the returned task + to find errors. This parameter has no effect if ``run_task_in_background`` + is ``True``. + + Returns: + The asyncio task object representing the background loop. + """ + task = asyncio.create_task( + self._call_periodically( + period, + func, + caller=self._find_caller(), + catch_errors=catch_errors, + run_task_in_background=run_task_in_background, + ) + ) + self._register_task(task) + return task + + def run_later( + self, delay: float | int, coro: Awaitable, catch_errors: bool = True + ) -> asyncio.Task: + """ + Run a coroutine after a delay. + + Examples: + >>> self.sched.run_later(5, self.async_task(meow=True)) + + Args: + delay: The delay in seconds to await the coroutine after. + coro: The coroutine to await. + catch_errors: Whether the scheduler should catch and log any errors. + If ``False``, errors will be raised, and the caller must await the returned task + to find errors. + + Returns: + The asyncio task object representing the scheduled task. + """ + task = asyncio.create_task( + self._call_with_delay( + delay, coro, caller=self._find_caller(), catch_errors=catch_errors + ) + ) + self._register_task(task) + return task + + def _register_task(self, task: asyncio.Task) -> None: + self.tasks.add(task) + task.add_done_callback(self.tasks.discard) + + async def _call_periodically( + self, + period: float | int, + func: Callable[[], Awaitable], + caller: str, + catch_errors: bool, + run_task_in_background: bool, + ) -> None: + while True: + try: + await asyncio.sleep(period) + if run_task_in_background: + self._register_task( + asyncio.create_task(self._call_periodically_background(func(), caller)) + ) + else: + await func() + except asyncio.CancelledError: + raise + except Exception: + if catch_errors: + self.log.exception(f"Uncaught error in background loop (created in {caller})") + else: + raise + + async def _call_periodically_background(self, coro: Awaitable, caller: str) -> None: + try: + await coro + except asyncio.CancelledError: + raise + except Exception: + self.log.exception(f"Uncaught error in background loop subtask (created in {caller})") + + async def _call_with_delay( + self, delay: float | int, coro: Awaitable, caller: str, catch_errors: bool + ) -> None: + try: + await asyncio.sleep(delay) + await coro + except asyncio.CancelledError: + raise + except Exception: + if catch_errors: + self.log.exception(f"Uncaught error in scheduled task (created in {caller})") + else: + raise + + def stop(self) -> None: + """ + Stop all scheduled tasks and background loops. + """ + for task in self.tasks: + task.cancel(msg="Scheduler stopped") From 3f2887d67f61920198f4476ae7300db80d0231f8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 7 Mar 2024 16:25:12 +0200 Subject: [PATCH 58/90] Update CI and pre-commit --- .github/workflows/python-lint.yml | 7 ++++--- .pre-commit-config.yaml | 6 +++--- dev-requirements.txt | 2 +- maubot/management/api/instance_database.py | 8 +++++--- pyproject.toml | 2 +- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index d88b8a8..b2fe29f 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -6,16 +6,17 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.12" - uses: isort/isort-action@master with: sortPaths: "./maubot" - uses: psf/black@stable with: src: "./maubot" + version: "24.2.0" - name: pre-commit run: | pip install pre-commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0de079e..e3c5bb8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: trailing-whitespace exclude_types: [markdown] @@ -8,13 +8,13 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 23.1.0 + rev: 24.2.0 hooks: - id: black language_version: python3 files: ^maubot/.*\.pyi?$ - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort files: ^maubot/.*\.pyi?$ diff --git a/dev-requirements.txt b/dev-requirements.txt index 5cd14c2..bb8c2a0 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,3 @@ pre-commit>=2.10.1,<3 isort>=5.10.1,<6 -black>=23,<24 +black>=24,<25 diff --git a/maubot/management/api/instance_database.py b/maubot/management/api/instance_database.py index 97b2edf..a40434f 100644 --- a/maubot/management/api/instance_database.py +++ b/maubot/management/api/instance_database.py @@ -56,9 +56,11 @@ async def get_table(request: web.Request) -> web.Response: try: order = [tuple(order.split(":")) for order in request.query.getall("order")] order = [ - (asc if sort.lower() == "asc" else desc)(table.columns[column]) - if sort - else table.columns[column] + ( + (asc if sort.lower() == "asc" else desc)(table.columns[column]) + if sort + else table.columns[column] + ) for column, sort in order ] except KeyError: diff --git a/pyproject.toml b/pyproject.toml index 4cee457..6c2ca27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,5 +9,5 @@ skip = ["maubot/management/frontend"] [tool.black] line-length = 99 -target-version = ["py38"] +target-version = ["py310"] force-exclude = "maubot/management/frontend" From 2c60342cc6c4d7e861060fb9d5a7244250fe6cd8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 10 Mar 2024 17:10:41 +0200 Subject: [PATCH 59/90] Update plugin list link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a2eda36..02a4b6f 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ All setup and usage instructions are located on Matrix room: [#maubot:maunium.net](https://matrix.to/#/#maubot:maunium.net) ## Plugins -A list of plugins can be found at [plugins.maubot.xyz](https://plugins.maubot.xyz/). +A list of plugins can be found at [plugins.mau.bot](https://plugins.mau.bot/). To add your plugin to the list, send a pull request to . From 7759643e935c6f7e21a3d2f4e8953d84d4c65f59 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 30 Mar 2024 23:31:40 +0200 Subject: [PATCH 60/90] Assume main class is in last module instead of first --- maubot/loader/zip.py | 2 +- maubot/standalone/__main__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/maubot/loader/zip.py b/maubot/loader/zip.py index 70cee5a..183d3e6 100644 --- a/maubot/loader/zip.py +++ b/maubot/loader/zip.py @@ -167,7 +167,7 @@ class ZippedPluginLoader(PluginLoader): if "/" in meta.main_class: self.main_module, self.main_class = meta.main_class.split("/")[:2] else: - self.main_module = meta.modules[0] + self.main_module = meta.modules[-1] self.main_class = meta.main_class self._file = file diff --git a/maubot/standalone/__main__.py b/maubot/standalone/__main__.py index 6d3150d..c320af4 100644 --- a/maubot/standalone/__main__.py +++ b/maubot/standalone/__main__.py @@ -115,7 +115,7 @@ with open(args.meta, "r") as meta_file: if "/" in meta.main_class: module, main_class = meta.main_class.split("/", 1) else: - module = meta.modules[0] + module = meta.modules[-1] main_class = meta.main_class if args.meta != "maubot.yaml" and os.path.dirname(args.meta) != "": From 4f68e20ff7c4f4efa5b1cf6f91a401f59c72f06e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 30 Mar 2024 23:31:48 +0200 Subject: [PATCH 61/90] Update changelog --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a3e691..f0ac9eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +# v0.5.0 (unreleased) + +* Fixed `main_class` to default to being loaded from the last module instead of + the first if a module name is not explicitly specified. + * This was already the [documented behavior](https://docs.mau.fi/maubot/dev/reference/plugin-metadata.html), + and loading from the first module doesn't make sense due to import order. +* Added simple scheduler utility for running background tasks periodically or + after a certain delay. +* Added testing framework for plugins (thanks to [@abompard] in [#225]). + +[#225]: https://github.com/maubot/maubot/issues/225 +[@abompard]: https://github.com/abompard + # v0.4.2 (2023-09-20) * Updated Pillow to 10.0.1. From a7f31f617527ccc9b594b732ebecd76d63bb9149 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 30 Mar 2024 23:32:08 +0200 Subject: [PATCH 62/90] Only include directories with __init__.py when building mbp file --- maubot/cli/commands/build.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/maubot/cli/commands/build.py b/maubot/cli/commands/build.py index ec3ac26..39eca53 100644 --- a/maubot/cli/commands/build.py +++ b/maubot/cli/commands/build.py @@ -93,10 +93,16 @@ def write_plugin(meta: PluginMeta, output: str | IO) -> None: if os.path.isfile(f"{module}.py"): zip.write(f"{module}.py") elif module is not None and os.path.isdir(module): - zipdir(zip, module) + if os.path.isfile(f"{module}/__init__.py"): + zipdir(zip, module) + else: + print( + Fore.YELLOW + + f"Module {module} is missing __init__.py, skipping" + + Fore.RESET + ) else: print(Fore.YELLOW + f"Module {module} not found, skipping" + Fore.RESET) - for pattern in meta.extra_files: for file in glob.iglob(pattern): zip.write(file) From 299d8f68c35f04ea1ed34a042709662b0be40d98 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 30 Mar 2024 23:36:54 +0200 Subject: [PATCH 63/90] Update changelog again --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0ac9eb..4e6ef24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ * Added simple scheduler utility for running background tasks periodically or after a certain delay. * Added testing framework for plugins (thanks to [@abompard] in [#225]). +* Changed `mbc build` to ignore directories declared in `modules` that are + missing an `__init__.py` file. + * Importing the modules at runtime would fail and break the plugin. + To include non-code resources outside modules in the mbp archive, + use `extra_files` instead. [#225]: https://github.com/maubot/maubot/issues/225 [@abompard]: https://github.com/abompard From 91f214819a1ac9cad4985342f60f1e0db0540f0a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 30 Mar 2024 23:37:07 +0200 Subject: [PATCH 64/90] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c57bb3c..9fd28ef 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ __pycache__ !example-config.yaml !.pre-commit-config.yaml +/start logs/ plugins/ trash/ From 861d81d2a6d74b12dbc60146422ccece79fc88c8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 13 Jul 2024 13:22:04 +0300 Subject: [PATCH 65/90] Update dependencies --- requirements.txt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4134190..30cf06e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,17 @@ -mautrix>=0.20.2,<0.21 +mautrix>=0.20.6,<0.21 aiohttp>=3,<4 yarl>=1,<2 SQLAlchemy>=1,<1.4 -asyncpg>=0.20,<0.29 -aiosqlite>=0.16,<0.19 +asyncpg>=0.20,<0.30 +aiosqlite>=0.16,<0.21 commonmark>=0.9,<1 -ruamel.yaml>=0.15.35,<0.18 +ruamel.yaml>=0.15.35,<0.19 attrs>=18.1.0 bcrypt>=3,<5 packaging>=10 click>=7,<9 colorama>=0.4,<0.5 -questionary>=1,<2 +questionary>=1,<3 jinja2>=2,<4 +setuptools From 09a0efbf19f885f90273f104943d2e04edb6869c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 6 Aug 2024 18:37:13 +0300 Subject: [PATCH 66/90] Remove hard dependency on SQLAlchemy Fixes #247 --- maubot/instance.py | 10 +++++----- maubot/lib/optionalalchemy.py | 19 +++++++++++++++++++ maubot/loader/zip.py | 6 +++--- maubot/management/api/instance_database.py | 12 ++++++------ maubot/management/api/plugin_upload.py | 17 ++++++++++++++--- maubot/management/api/responses.py | 15 ++++++++++++++- maubot/plugin_base.py | 5 +++-- optional-requirements.txt | 3 +++ requirements.txt | 1 - 9 files changed, 67 insertions(+), 21 deletions(-) create mode 100644 maubot/lib/optionalalchemy.py diff --git a/maubot/instance.py b/maubot/instance.py index 2905d12..8427e3c 100644 --- a/maubot/instance.py +++ b/maubot/instance.py @@ -25,7 +25,6 @@ import os.path from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap -import sqlalchemy as sql from mautrix.types import UserID from mautrix.util import background_task @@ -36,6 +35,7 @@ from mautrix.util.logging import TraceLogger from .client import Client from .db import DatabaseEngine, Instance as DBInstance +from .lib.optionalalchemy import Engine, MetaData, create_engine from .lib.plugin_db import ProxyPostgresDatabase from .loader import DatabaseType, PluginLoader, ZippedPluginLoader from .plugin_base import Plugin @@ -128,7 +128,7 @@ class PluginInstance(DBInstance): } def _introspect_sqlalchemy(self) -> dict: - metadata = sql.MetaData() + metadata = MetaData() metadata.reflect(self.inst_db) return { table.name: { @@ -214,7 +214,7 @@ class PluginInstance(DBInstance): async def get_db_tables(self) -> dict: if self.inst_db_tables is None: - if isinstance(self.inst_db, sql.engine.Engine): + if isinstance(self.inst_db, Engine): self.inst_db_tables = self._introspect_sqlalchemy() elif self.inst_db.scheme == Scheme.SQLITE: self.inst_db_tables = await self._introspect_sqlite() @@ -294,7 +294,7 @@ class PluginInstance(DBInstance): "Instance database engine is marked as Postgres, but plugin uses legacy " "database interface, which doesn't support postgres." ) - self.inst_db = sql.create_engine(f"sqlite:///{self._sqlite_db_path}") + self.inst_db = create_engine(f"sqlite:///{self._sqlite_db_path}") elif self.loader.meta.database_type == DatabaseType.ASYNCPG: if self.database_engine is None: if os.path.exists(self._sqlite_db_path) or not self.maubot.plugin_postgres_db: @@ -329,7 +329,7 @@ class PluginInstance(DBInstance): async def stop_database(self) -> None: if isinstance(self.inst_db, Database): await self.inst_db.stop() - elif isinstance(self.inst_db, sql.engine.Engine): + elif isinstance(self.inst_db, Engine): self.inst_db.dispose() else: raise RuntimeError(f"Unknown database type {type(self.inst_db).__name__}") diff --git a/maubot/lib/optionalalchemy.py b/maubot/lib/optionalalchemy.py new file mode 100644 index 0000000..ba94271 --- /dev/null +++ b/maubot/lib/optionalalchemy.py @@ -0,0 +1,19 @@ +try: + from sqlalchemy import MetaData, asc, create_engine, desc + from sqlalchemy.engine import Engine + from sqlalchemy.exc import IntegrityError, OperationalError +except ImportError: + + class FakeError(Exception): + pass + + class FakeType: + def __init__(self, *args, **kwargs): + raise Exception("SQLAlchemy is not installed") + + def create_engine(*args, **kwargs): + raise Exception("SQLAlchemy is not installed") + + MetaData = Engine = FakeType + IntegrityError = OperationalError = FakeError + asc = desc = lambda a: a diff --git a/maubot/loader/zip.py b/maubot/loader/zip.py index 183d3e6..8642183 100644 --- a/maubot/loader/zip.py +++ b/maubot/loader/zip.py @@ -31,7 +31,7 @@ from ..config import Config from ..lib.zipimport import ZipImportError, zipimporter from ..plugin_base import Plugin from .abc import IDConflictError, PluginClass, PluginLoader -from .meta import PluginMeta +from .meta import DatabaseType, PluginMeta current_version = Version(__version__) yaml = YAML() @@ -155,9 +155,9 @@ class ZippedPluginLoader(PluginLoader): return file, meta @classmethod - def verify_meta(cls, source) -> tuple[str, Version]: + def verify_meta(cls, source) -> tuple[str, Version, DatabaseType | None]: _, meta = cls._read_meta(source) - return meta.id, meta.version + return meta.id, meta.version, meta.database_type if meta.database else None def _load_meta(self) -> None: file, meta = self._read_meta(self.path) diff --git a/maubot/management/api/instance_database.py b/maubot/management/api/instance_database.py index a40434f..2f8c37a 100644 --- a/maubot/management/api/instance_database.py +++ b/maubot/management/api/instance_database.py @@ -19,12 +19,12 @@ from datetime import datetime from aiohttp import web from asyncpg import PostgresError -from sqlalchemy import asc, desc, engine, exc import aiosqlite from mautrix.util.async_db import Database from ...instance import PluginInstance +from ...lib.optionalalchemy import Engine, IntegrityError, OperationalError, asc, desc from .base import routes from .responses import resp @@ -66,7 +66,7 @@ async def get_table(request: web.Request) -> web.Response: except KeyError: order = [] limit = int(request.query.get("limit", "100")) - if isinstance(instance.inst_db, engine.Engine): + if isinstance(instance.inst_db, Engine): return _execute_query_sqlalchemy(instance, table.select().order_by(*order).limit(limit)) @@ -84,7 +84,7 @@ async def query(request: web.Request) -> web.Response: except KeyError: return resp.query_missing rows_as_dict = data.get("rows_as_dict", False) - if isinstance(instance.inst_db, engine.Engine): + if isinstance(instance.inst_db, Engine): return _execute_query_sqlalchemy(instance, sql_query, rows_as_dict) elif isinstance(instance.inst_db, Database): try: @@ -133,12 +133,12 @@ async def _execute_query_asyncpg( def _execute_query_sqlalchemy( instance: PluginInstance, sql_query: str, rows_as_dict: bool = False ) -> web.Response: - assert isinstance(instance.inst_db, engine.Engine) + assert isinstance(instance.inst_db, Engine) try: res = instance.inst_db.execute(sql_query) - except exc.IntegrityError as e: + except IntegrityError as e: return resp.sql_integrity_error(e, sql_query) - except exc.OperationalError as e: + except OperationalError as e: return resp.sql_operational_error(e, sql_query) data = { "ok": True, diff --git a/maubot/management/api/plugin_upload.py b/maubot/management/api/plugin_upload.py index ea4fd1f..4cd2c47 100644 --- a/maubot/management/api/plugin_upload.py +++ b/maubot/management/api/plugin_upload.py @@ -23,10 +23,17 @@ import traceback from aiohttp import web from packaging.version import Version -from ...loader import MaubotZipImportError, PluginLoader, ZippedPluginLoader +from ...loader import DatabaseType, MaubotZipImportError, PluginLoader, ZippedPluginLoader from .base import get_config, routes from .responses import resp +try: + import sqlalchemy + + has_alchemy = True +except ImportError: + has_alchemy = False + log = logging.getLogger("maubot.server.upload") @@ -36,9 +43,11 @@ async def put_plugin(request: web.Request) -> web.Response: content = await request.read() file = BytesIO(content) try: - pid, version = ZippedPluginLoader.verify_meta(file) + pid, version, db_type = ZippedPluginLoader.verify_meta(file) except MaubotZipImportError as e: return resp.plugin_import_error(str(e), traceback.format_exc()) + if db_type == DatabaseType.SQLALCHEMY and not has_alchemy: + return resp.sqlalchemy_not_installed if pid != plugin_id: return resp.pid_mismatch plugin = PluginLoader.id_cache.get(plugin_id, None) @@ -55,9 +64,11 @@ async def upload_plugin(request: web.Request) -> web.Response: content = await request.read() file = BytesIO(content) try: - pid, version = ZippedPluginLoader.verify_meta(file) + pid, version, db_type = ZippedPluginLoader.verify_meta(file) except MaubotZipImportError as e: return resp.plugin_import_error(str(e), traceback.format_exc()) + if db_type == DatabaseType.SQLALCHEMY and not has_alchemy: + return resp.sqlalchemy_not_installed plugin = PluginLoader.id_cache.get(pid, None) if not plugin: return await upload_new_plugin(content, pid, version) diff --git a/maubot/management/api/responses.py b/maubot/management/api/responses.py index 15f6a96..0f22caa 100644 --- a/maubot/management/api/responses.py +++ b/maubot/management/api/responses.py @@ -15,13 +15,16 @@ # along with this program. If not, see . from __future__ import annotations +from typing import TYPE_CHECKING from http import HTTPStatus from aiohttp import web from asyncpg import PostgresError -from sqlalchemy.exc import IntegrityError, OperationalError import aiosqlite +if TYPE_CHECKING: + from sqlalchemy.exc import IntegrityError, OperationalError + class _Response: @property @@ -324,6 +327,16 @@ class _Response: } ) + @property + def sqlalchemy_not_installed(self) -> web.Response: + return web.json_response( + { + "error": "This plugin requires a legacy database, but SQLAlchemy is not installed", + "errcode": "unsupported_plugin_database", + }, + status=HTTPStatus.NOT_IMPLEMENTED, + ) + @property def table_not_found(self) -> web.Response: return web.json_response( diff --git a/maubot/plugin_base.py b/maubot/plugin_base.py index 5e967dc..1be15e0 100644 --- a/maubot/plugin_base.py +++ b/maubot/plugin_base.py @@ -20,7 +20,6 @@ from abc import ABC from asyncio import AbstractEventLoop from aiohttp import ClientSession -from sqlalchemy.engine.base import Engine from yarl import URL from mautrix.util.async_db import Database, UpgradeTable @@ -30,6 +29,8 @@ from mautrix.util.logging import TraceLogger from .scheduler import BasicScheduler if TYPE_CHECKING: + from sqlalchemy.engine.base import Engine + from .client import MaubotMatrixClient from .loader import BasePluginLoader from .plugin_server import PluginWebApp @@ -56,7 +57,7 @@ class Plugin(ABC): instance_id: str, log: TraceLogger, config: BaseProxyConfig | None, - database: Engine | None, + database: Engine | Database | None, webapp: PluginWebApp | None, webapp_url: str | None, loader: BasePluginLoader, diff --git a/optional-requirements.txt b/optional-requirements.txt index 0e45b97..f5b378a 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -9,3 +9,6 @@ unpaddedbase64>=1,<3 #/testing pytest pytest-asyncio + +#/legacydb +SQLAlchemy>1,<1.4 diff --git a/requirements.txt b/requirements.txt index 30cf06e..7de02dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ mautrix>=0.20.6,<0.21 aiohttp>=3,<4 yarl>=1,<2 -SQLAlchemy>=1,<1.4 asyncpg>=0.20,<0.30 aiosqlite>=0.16,<0.21 commonmark>=0.9,<1 From 49adb9b4412a409b385bcdce84f72f057b32e7c4 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 6 Aug 2024 18:51:36 +0300 Subject: [PATCH 67/90] Update docker image --- .gitlab-ci.yml | 2 +- Dockerfile | 12 +++++------- Dockerfile.ci | 10 ++++------ 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0c5196e..8c8c6db 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,7 @@ default: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY build frontend: - image: node:18-alpine + image: node:20-alpine stage: build frontend before_script: [] variables: diff --git a/Dockerfile b/Dockerfile index f4bd0df..b4ee106 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ -FROM node:18 AS frontend-builder +FROM node:20 AS frontend-builder COPY ./maubot/management/frontend /frontend RUN cd /frontend && yarn --prod && yarn build -FROM alpine:3.18 +FROM alpine:3.20 RUN apk add --no-cache \ python3 py3-pip py3-setuptools py3-wheel \ @@ -11,7 +11,6 @@ RUN apk add --no-cache \ su-exec \ yq \ py3-aiohttp \ - py3-sqlalchemy \ py3-attrs \ py3-bcrypt \ py3-cffi \ @@ -34,20 +33,19 @@ RUN apk add --no-cache \ py3-unpaddedbase64 \ py3-future \ # plugin deps - #py3-pillow \ + py3-pillow \ py3-magic \ py3-feedparser \ py3-dateutil \ py3-lxml \ - py3-semver \ - && apk add --no-cache py3-pillow --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community + py3-semver # TODO remove pillow, magic, feedparser, lxml, gitlab and semver when maubot supports installing dependencies COPY requirements.txt /opt/maubot/requirements.txt COPY optional-requirements.txt /opt/maubot/optional-requirements.txt WORKDIR /opt/maubot RUN apk add --virtual .build-deps python3-dev build-base git \ - && pip3 install -r requirements.txt -r optional-requirements.txt \ + && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \ dateparser langdetect python-gitlab pyquery tzlocal \ && apk del .build-deps # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies diff --git a/Dockerfile.ci b/Dockerfile.ci index 7a957f8..3f83a1c 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM alpine:3.18 +FROM alpine:3.20 RUN apk add --no-cache \ python3 py3-pip py3-setuptools py3-wheel \ @@ -6,7 +6,6 @@ RUN apk add --no-cache \ su-exec \ yq \ py3-aiohttp \ - py3-sqlalchemy \ py3-attrs \ py3-bcrypt \ py3-cffi \ @@ -30,11 +29,10 @@ RUN apk add --no-cache \ py3-unpaddedbase64 \ py3-future \ # plugin deps - #py3-pillow \ + py3-pillow \ py3-magic \ py3-feedparser \ - py3-lxml \ - && apk add --no-cache py3-pillow --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community + py3-lxml # py3-gitlab # py3-semver # TODO remove pillow, magic, feedparser, lxml, gitlab and semver when maubot supports installing dependencies @@ -43,7 +41,7 @@ COPY requirements.txt /opt/maubot/requirements.txt COPY optional-requirements.txt /opt/maubot/optional-requirements.txt WORKDIR /opt/maubot RUN apk add --virtual .build-deps python3-dev build-base git \ - && pip3 install -r requirements.txt -r optional-requirements.txt \ + && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \ dateparser langdetect python-gitlab pyquery semver tzlocal cssselect \ && apk del .build-deps # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies From b8714cc6b9cd0fc484fdcbb4805f51ddd1fb689f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 6 Aug 2024 18:55:00 +0300 Subject: [PATCH 68/90] Also update standalone docker image --- maubot/standalone/Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/maubot/standalone/Dockerfile b/maubot/standalone/Dockerfile index 2db8426..8cfa8cd 100644 --- a/maubot/standalone/Dockerfile +++ b/maubot/standalone/Dockerfile @@ -1,9 +1,8 @@ -FROM docker.io/alpine:3.18 +FROM docker.io/alpine:3.20 RUN apk add --no-cache \ python3 py3-pip py3-setuptools py3-wheel \ py3-aiohttp \ - py3-sqlalchemy \ py3-attrs \ py3-bcrypt \ py3-cffi \ @@ -26,8 +25,8 @@ RUN cd /opt/maubot \ python3-dev \ libffi-dev \ build-base \ - && pip3 install -r requirements.txt -r optional-requirements.txt \ + && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \ && apk del .build-deps COPY . /opt/maubot -RUN cd /opt/maubot && pip3 install . +RUN cd /opt/maubot && pip3 install --break-system-packages . From c218c8cf6198e915c1df55958286c98557fd56e7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 24 Aug 2024 12:10:18 +0300 Subject: [PATCH 69/90] Bump version to 0.5.0 --- CHANGELOG.md | 6 +++++- maubot/__meta__.py | 2 +- setup.py | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e6ef24..b112eb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ -# v0.5.0 (unreleased) +# v0.5.0 (2024-08-24) +* Dropped Python 3.9 support. +* Updated Docker image to Alpine 3.20. +* Updated mautrix-python to 0.20.6 to support authenticated media. +* Removed hard dependency on SQLAlchemy. * Fixed `main_class` to default to being loaded from the last module instead of the first if a module name is not explicitly specified. * This was already the [documented behavior](https://docs.mau.fi/maubot/dev/reference/plugin-metadata.html), diff --git a/maubot/__meta__.py b/maubot/__meta__.py index df12433..3d18726 100644 --- a/maubot/__meta__.py +++ b/maubot/__meta__.py @@ -1 +1 @@ -__version__ = "0.4.2" +__version__ = "0.5.0" diff --git a/setup.py b/setup.py index 79a0c6c..cf16f7e 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setuptools.setup( install_requires=install_requires, extras_require=extras_require, - python_requires="~=3.9", + python_requires="~=3.10", classifiers=[ "Development Status :: 4 - Beta", @@ -50,9 +50,9 @@ setuptools.setup( "Framework :: AsyncIO", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ], entry_points=""" [console_scripts] From 65be63fdd251b82733c1d95906c077bb22a79ef2 Mon Sep 17 00:00:00 2001 From: jkhsjdhjs Date: Sat, 24 Aug 2024 17:47:24 +0200 Subject: [PATCH 70/90] Fix PluginWebApp base path handling (#240) Previously, the webapp handler would match without respect to the trailing slash, e.g. matching "foo" for "foo2". This behavior is changed to respect the trailing slash. Fixes #239 --- maubot/server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/maubot/server.py b/maubot/server.py index 097fe5b..dd1101e 100644 --- a/maubot/server.py +++ b/maubot/server.py @@ -64,14 +64,14 @@ class MaubotServer: if request.path.startswith(path): request = request.clone( rel_url=request.rel_url.with_path( - request.rel_url.path[len(path) :] + request.rel_url.path[len(path) - 1 :] ).with_query(request.query_string) ) return await app.handle(request) return web.Response(status=404) def get_instance_subapp(self, instance_id: str) -> tuple[PluginWebApp, str]: - subpath = self.config["server.plugin_base_path"] + instance_id + subpath = self.config["server.plugin_base_path"] + instance_id + "/" url = self.config["server.public_url"] + subpath try: return self.plugin_routes[subpath], url @@ -82,7 +82,7 @@ class MaubotServer: def remove_instance_webapp(self, instance_id: str) -> None: try: - subpath = self.config["server.plugin_base_path"] + instance_id + subpath = self.config["server.plugin_base_path"] + instance_id + "/" self.plugin_routes.pop(subpath).clear() except KeyError: return From 472fb9f6acbb0e93c2d819d17077c25548aca284 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 8 Sep 2024 00:58:36 +0300 Subject: [PATCH 71/90] Remove outdated comment [skip ci] --- maubot/example-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/maubot/example-config.yaml b/maubot/example-config.yaml index a16ea19..f21ac1e 100644 --- a/maubot/example-config.yaml +++ b/maubot/example-config.yaml @@ -1,5 +1,4 @@ # The full URI to the database. SQLite and Postgres are fully supported. -# Other DBMSes supported by SQLAlchemy may or may not work. # Format examples: # SQLite: sqlite:filename.db # Postgres: postgresql://username:password@hostname/dbname From dd58135c94d1a312e7ebad82815c41b638395e17 Mon Sep 17 00:00:00 2001 From: Dominik Rimpf Date: Thu, 3 Oct 2024 23:59:34 +0200 Subject: [PATCH 72/90] Update media endpoints in management frontend (#253) --- maubot/management/frontend/src/api.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/maubot/management/frontend/src/api.js b/maubot/management/frontend/src/api.js index 5c1fd55..b173e4c 100644 --- a/maubot/management/frontend/src/api.js +++ b/maubot/management/frontend/src/api.js @@ -205,7 +205,7 @@ export const getClients = () => defaultGet("/clients") export const getClient = id => defaultGet(`/clients/${id}`) export async function uploadAvatar(id, data, mime) { - const resp = await fetch(`${BASE_PATH}/proxy/${id}/_matrix/media/r0/upload`, { + const resp = await fetch(`${BASE_PATH}/proxy/${id}/_matrix/media/v3/upload`, { headers: getHeaders(mime), body: data, method: "POST", @@ -217,8 +217,8 @@ export function getAvatarURL({ id, avatar_url }) { if (!avatar_url?.startsWith("mxc://")) { return null } - avatar_url = avatar_url.substr("mxc://".length) - return `${BASE_PATH}/proxy/${id}/_matrix/media/r0/download/${avatar_url}?access_token=${ + avatar_url = avatar_url.substring("mxc://".length) + return `${BASE_PATH}/proxy/${id}/_matrix/client/v1/media/download/${avatar_url}?access_token=${ localStorage.accessToken}` } From bceacb97a0a862fb9a626959a83442c616c0c65f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 4 Oct 2024 00:58:15 +0300 Subject: [PATCH 73/90] Cut off plaintext body if the event is too long --- maubot/matrix.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/maubot/matrix.py b/maubot/matrix.py index 0f3e36b..87c7b70 100644 --- a/maubot/matrix.py +++ b/maubot/matrix.py @@ -62,7 +62,10 @@ async def parse_formatted( html = message else: return message, escape(message) - return (await MaubotHTMLParser().parse(html)).text, html + text = (await MaubotHTMLParser().parse(html)).text + if len(text) + len(html) > 60000: + text = text[:100] + "[long message cut off]" + return text, html class MaubotMessageEvent(MessageEvent): From 48cc00f5910a43a09ef09e4393f638331294c11b Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Thu, 2 Jan 2025 09:18:06 +0000 Subject: [PATCH 74/90] Update asyncpg dependency to fix python 3.13 support (#256) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7de02dd..5a4312d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ mautrix>=0.20.6,<0.21 aiohttp>=3,<4 yarl>=1,<2 -asyncpg>=0.20,<0.30 +asyncpg>=0.20,<0.31 aiosqlite>=0.16,<0.21 commonmark>=0.9,<1 ruamel.yaml>=0.15.35,<0.19 From 46aed7e1d2bbf1e7a161c8ce4cb2c8a50356c229 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 3 Jan 2025 12:26:25 +0200 Subject: [PATCH 75/90] Relax asyncpg and aiosqlite version requirement --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5a4312d..e1df001 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -mautrix>=0.20.6,<0.21 +mautrix>=0.20.7,<0.21 aiohttp>=3,<4 yarl>=1,<2 -asyncpg>=0.20,<0.31 -aiosqlite>=0.16,<0.21 +asyncpg>=0.20,<1 +aiosqlite>=0.16,<1 commonmark>=0.9,<1 ruamel.yaml>=0.15.35,<0.19 attrs>=18.1.0 From 813fee7a2ca8e945d75e10887e65a4451b9c076d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 3 Jan 2025 12:26:51 +0200 Subject: [PATCH 76/90] Update linters --- .github/workflows/python-lint.yml | 4 ++-- .pre-commit-config.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index b2fe29f..28d6df2 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -9,14 +9,14 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - uses: isort/isort-action@master with: sortPaths: "./maubot" - uses: psf/black@stable with: src: "./maubot" - version: "24.2.0" + version: "24.10.0" - name: pre-commit run: | pip install pre-commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e3c5bb8..4a6328e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v5.0.0 hooks: - id: trailing-whitespace exclude_types: [markdown] @@ -8,7 +8,7 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 24.2.0 + rev: 24.10.0 hooks: - id: black language_version: python3 From 01b5f53d906c917db2bf1de06d757468cc75b68d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 3 Jan 2025 12:31:01 +0200 Subject: [PATCH 77/90] Update Alpine and Node --- .gitlab-ci.yml | 2 +- Dockerfile | 4 ++-- Dockerfile.ci | 2 +- maubot/standalone/Dockerfile | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8c8c6db..50d0c15 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,7 @@ default: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY build frontend: - image: node:20-alpine + image: node:22-alpine stage: build frontend before_script: [] variables: diff --git a/Dockerfile b/Dockerfile index b4ee106..2c6bad4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ -FROM node:20 AS frontend-builder +FROM node:22 AS frontend-builder COPY ./maubot/management/frontend /frontend RUN cd /frontend && yarn --prod && yarn build -FROM alpine:3.20 +FROM alpine:3.21 RUN apk add --no-cache \ python3 py3-pip py3-setuptools py3-wheel \ diff --git a/Dockerfile.ci b/Dockerfile.ci index 3f83a1c..9712a16 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM alpine:3.20 +FROM alpine:3.21 RUN apk add --no-cache \ python3 py3-pip py3-setuptools py3-wheel \ diff --git a/maubot/standalone/Dockerfile b/maubot/standalone/Dockerfile index 8cfa8cd..54623f2 100644 --- a/maubot/standalone/Dockerfile +++ b/maubot/standalone/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/alpine:3.20 +FROM docker.io/alpine:3.21 RUN apk add --no-cache \ python3 py3-pip py3-setuptools py3-wheel \ From 6c7d0754f80bead267bbcf126d5b9e58460febcb Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 3 Jan 2025 12:32:04 +0200 Subject: [PATCH 78/90] Add Python 3.13 to classifiers --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index cf16f7e..838196f 100644 --- a/setup.py +++ b/setup.py @@ -53,6 +53,7 @@ setuptools.setup( "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ], entry_points=""" [console_scripts] From c3458eab5808ab1fa911eff902750064fdfedcce Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 3 Jan 2025 12:40:46 +0200 Subject: [PATCH 79/90] Bump version to 0.5.1 --- CHANGELOG.md | 16 ++++++++++++++++ maubot/__meta__.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b112eb2..3a04252 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +# v0.5.1 (2025-01-03) + +* Updated Docker image to Alpine 3.21. +* Updated media upload/download endpoints in management frontend + (thanks to [@domrim] in [#253]). +* Fixed plugin web app base path not including a trailing slash + (thanks to [@jkhsjdhjs] in [#240]). +* Changed markdown parsing to cut off plaintext body if necessary to allow + longer formatted messages. +* Updated dependencies to fix Python 3.13 compatibility. + +[@domrim]: https://github.com/domrim +[@jkhsjdhjs]: https://github.com/jkhsjdhjs +[#253]: https://github.com/maubot/maubot/pull/253 +[#240]: https://github.com/maubot/maubot/pull/240 + # v0.5.0 (2024-08-24) * Dropped Python 3.9 support. diff --git a/maubot/__meta__.py b/maubot/__meta__.py index 3d18726..dd9b22c 100644 --- a/maubot/__meta__.py +++ b/maubot/__meta__.py @@ -1 +1 @@ -__version__ = "0.5.0" +__version__ = "0.5.1" From 094e1eca35fd7d859bdf03db0555925986265996 Mon Sep 17 00:00:00 2001 From: Binesh Bannerjee Date: Wed, 22 Jan 2025 13:10:39 -0500 Subject: [PATCH 80/90] Fix autojoin and online flags not being applied if set during client creation (#258) --- maubot/management/api/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maubot/management/api/client.py b/maubot/management/api/client.py index 2a8964c..d2ad35d 100644 --- a/maubot/management/api/client.py +++ b/maubot/management/api/client.py @@ -78,8 +78,8 @@ async def _create_client(user_id: UserID | None, data: dict) -> web.Response: ) client.enabled = data.get("enabled", True) client.sync = data.get("sync", True) - client.autojoin = data.get("autojoin", True) - client.online = data.get("online", True) + await client.update_autojoin(data.get("autojoin", True), save=False) + await client.update_online(data.get("online", True), save=False) client.displayname = data.get("displayname", "disable") client.avatar_url = data.get("avatar_url", "disable") await client.update() From c09eb195f87a022e0530b4bd8974cd4cfb1eca01 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 28 Jan 2025 16:55:18 +0200 Subject: [PATCH 81/90] Add comment --- maubot/management/frontend/src/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/maubot/management/frontend/src/api.js b/maubot/management/frontend/src/api.js index b173e4c..d1a51b8 100644 --- a/maubot/management/frontend/src/api.js +++ b/maubot/management/frontend/src/api.js @@ -218,6 +218,7 @@ export function getAvatarURL({ id, avatar_url }) { return null } avatar_url = avatar_url.substring("mxc://".length) + // Note: the maubot backend will replace the query param with an authorization header return `${BASE_PATH}/proxy/${id}/_matrix/client/v1/media/download/${avatar_url}?access_token=${ localStorage.accessToken}` } From fe4d2f02bb8ad8ee37a46777a968fb5988bc7737 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 28 Jan 2025 16:55:42 +0200 Subject: [PATCH 82/90] Fix clearing PluginWebApp Fixes #233 --- maubot/plugin_server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/maubot/plugin_server.py b/maubot/plugin_server.py index 9dd2df4..e5c246c 100644 --- a/maubot/plugin_server.py +++ b/maubot/plugin_server.py @@ -40,6 +40,8 @@ class PluginWebApp(web.UrlDispatcher): self._resources = [] self._named_resources = {} self._middleware = [] + self._resource_index = {} + self._matched_sub_app_resources = [] async def handle(self, request: web.Request) -> web.StreamResponse: match_info = await self.resolve(request) From f0ade0a04333fe8b347a4781474d0cb397c5c39f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 4 May 2025 00:43:35 +0300 Subject: [PATCH 83/90] Clarify type of admins map [skip ci] --- maubot/example-config.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/maubot/example-config.yaml b/maubot/example-config.yaml index f21ac1e..0a6c8ac 100644 --- a/maubot/example-config.yaml +++ b/maubot/example-config.yaml @@ -78,8 +78,9 @@ homeservers: # When this is empty, `mbc auth --register` won't work, but `mbc auth` (login) will. secret: null -# List of administrator users. Plaintext passwords will be bcrypted on startup. Set empty password -# to prevent normal login. Root is a special user that can't have a password and will always exist. +# List of administrator users. Each key is a username and the value is the password. +# Plaintext passwords will be bcrypted on startup. Set empty password to prevent normal login. +# Root is a special user that can't have a password and will always exist. admins: root: "" From 80b65d6a2f69fda8500c570e3836e2ac2e8815ad Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 5 May 2025 00:59:20 +0300 Subject: [PATCH 84/90] Improve tombstone handling --- maubot/client.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/maubot/client.py b/maubot/client.py index bdb76fc..f06af83 100644 --- a/maubot/client.py +++ b/maubot/client.py @@ -353,8 +353,33 @@ class Client(DBClient): if not evt.content.replacement_room: self.log.info(f"{evt.room_id} tombstoned with no replacement, ignoring") return + is_joined = await self.client.state_store.is_joined( + evt.content.replacement_room, + self.client.mxid, + ) + if is_joined: + self.log.debug( + f"Ignoring tombstone from {evt.room_id} to {evt.content.replacement_room} " + f"sent by {evt.sender}: already joined to replacement room" + ) + return + self.log.debug( + f"Following tombstone from {evt.room_id} to {evt.content.replacement_room} " + f"sent by {evt.sender}" + ) _, server = self.client.parse_user_id(evt.sender) - await self.client.join_room(evt.content.replacement_room, servers=[server]) + room_id = await self.client.join_room(evt.content.replacement_room, servers=[server]) + power_levels = await self.client.get_state_event(room_id, EventType.ROOM_POWER_LEVELS) + if power_levels.get_user_level(evt.sender) < power_levels.invite: + self.log.warning( + f"{evt.room_id} was tombstoned into {room_id}," + " but the sender doesn't have invite power levels, leaving..." + ) + await self.client.leave_room( + room_id, + f"Followed tombstone from {evt.room_id} by {evt.sender}," + " but sender doesn't have sufficient power level for invites", + ) async def _handle_invite(self, evt: StrippedStateEvent) -> None: if evt.state_key == self.id and evt.content.membership == Membership.INVITE: From 59cfff99f152aa92c2ec564de32218930b1baf78 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 5 May 2025 01:29:43 +0300 Subject: [PATCH 85/90] Adjust log --- maubot/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maubot/client.py b/maubot/client.py index f06af83..56efbbb 100644 --- a/maubot/client.py +++ b/maubot/client.py @@ -372,7 +372,7 @@ class Client(DBClient): power_levels = await self.client.get_state_event(room_id, EventType.ROOM_POWER_LEVELS) if power_levels.get_user_level(evt.sender) < power_levels.invite: self.log.warning( - f"{evt.room_id} was tombstoned into {room_id}," + f"{evt.room_id} was tombstoned into {room_id} by {evt.sender}," " but the sender doesn't have invite power levels, leaving..." ) await self.client.leave_room( From 9109047ef25bc556d96715e10cac2e45599e2ffa Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 6 May 2025 00:10:47 +0300 Subject: [PATCH 86/90] Bump version to 0.5.2 --- CHANGELOG.md | 11 +++++++++++ maubot/__meta__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a04252..d9de2b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +# v0.5.2 (2025-05-05) + +* Improved tombstone handling to ensure that the tombstone sender has + permissions to invite users to the target room. +* Fixed autojoin and online flags not being applied if set during client + creation (thanks to [@bnsh] in [#258]). +* Fixed plugin web apps not being cleared properly when unloading plugins. + +[@bnsh]: https://github.com/bnsh +[#258]: https://github.com/maubot/maubot/pull/258 + # v0.5.1 (2025-01-03) * Updated Docker image to Alpine 3.21. diff --git a/maubot/__meta__.py b/maubot/__meta__.py index dd9b22c..7225152 100644 --- a/maubot/__meta__.py +++ b/maubot/__meta__.py @@ -1 +1 @@ -__version__ = "0.5.1" +__version__ = "0.5.2" From ac3f0c34ccbe40da91bc4955ab63d3a2b4d22f52 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 14 May 2025 17:18:02 +0300 Subject: [PATCH 87/90] Reduce limit when plaintext body is cut off --- maubot/matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maubot/matrix.py b/maubot/matrix.py index 87c7b70..a99c1a4 100644 --- a/maubot/matrix.py +++ b/maubot/matrix.py @@ -63,7 +63,7 @@ async def parse_formatted( else: return message, escape(message) text = (await MaubotHTMLParser().parse(html)).text - if len(text) + len(html) > 60000: + if len(text) + len(html) > 40000: text = text[:100] + "[long message cut off]" return text, html From 10383d526f84f8d56c7e24833d510f36953006b1 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 14 May 2025 17:18:18 +0300 Subject: [PATCH 88/90] Ignore tombstones with non-empty state key --- maubot/client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/maubot/client.py b/maubot/client.py index 56efbbb..b0fde73 100644 --- a/maubot/client.py +++ b/maubot/client.py @@ -350,6 +350,8 @@ class Client(DBClient): } async def _handle_tombstone(self, evt: StateEvent) -> None: + if evt.state_key != "": + return if not evt.content.replacement_room: self.log.info(f"{evt.room_id} tombstoned with no replacement, ignoring") return From 51e1f699add4bc57f21d33c9f0381bed0a666826 Mon Sep 17 00:00:00 2001 From: vbatts Date: Fri, 1 Aug 2025 09:01:34 -0400 Subject: [PATCH 89/90] vb: changes for running locally with my own docker image --- Dockerfile.local | 30 ++++++++++++++++++++++++++++++ docker/run.sh | 6 +++++- 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 Dockerfile.local diff --git a/Dockerfile.local b/Dockerfile.local new file mode 100644 index 0000000..30372f2 --- /dev/null +++ b/Dockerfile.local @@ -0,0 +1,30 @@ +FROM r.batts.cloud/nodejs:18 AS frontend-builder + +COPY ./maubot/management/frontend /frontend +RUN cd /frontend && yarn --prod && yarn build + +#FROM alpine:3.18 +FROM r.batts.cloud/debian:bookworm + +RUN apt update && \ + apt install -y --no-install-recommends python3 python3-dev python3-venv python3-semver git gosu yq brotli && \ + apt clean -y && \ + rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /opt/maubot/requirements.txt +COPY optional-requirements.txt /opt/maubot/optional-requirements.txt +WORKDIR /opt/maubot +RUN python3 -m venv /venv \ + && bash -c 'source /venv/bin/activate \ + && pip3 install -r requirements.txt -r optional-requirements.txt \ + dateparser langdetect python-gitlab pyquery tzlocal pyfiglet emoji feedparser brotli' +# TODO also remove pyfiglet, emoji, dateparser, langdetect and pyquery when maubot supports installing dependencies + +COPY . /opt/maubot +RUN cp maubot/example-config.yaml . +COPY ./docker/mbc.sh /usr/local/bin/mbc +COPY --from=frontend-builder /frontend/build /opt/maubot/frontend +ENV UID=1337 GID=1337 XDG_CONFIG_HOME=/data +VOLUME /data + +CMD ["/opt/maubot/docker/run.sh"] diff --git a/docker/run.sh b/docker/run.sh index 5447e27..1ec95a2 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash function fixperms { chown -R $UID:$GID /var/log /data @@ -43,4 +43,8 @@ if ls /data/plugins/*.db > /dev/null 2>&1; then mv -n /data/plugins/*.db /data/dbs/ fi +if [ -f "/venv/bin/activate" ] ; then + exec gosu $UID:$GID bash -c 'source /venv/bin/activate && python3 -m maubot -c /data/config.yaml' +fi exec su-exec $UID:$GID python3 -m maubot -c /data/config.yaml + From 4aab38eb776cca560381963c9cb1c43f27be5629 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Fri, 1 Aug 2025 09:55:20 -0400 Subject: [PATCH 90/90] vb: update nodejs in local build --- Dockerfile.local | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile.local b/Dockerfile.local index 30372f2..d37e220 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -1,9 +1,8 @@ -FROM r.batts.cloud/nodejs:18 AS frontend-builder +FROM r.batts.cloud/nodejs:22 AS frontend-builder COPY ./maubot/management/frontend /frontend RUN cd /frontend && yarn --prod && yarn build -#FROM alpine:3.18 FROM r.batts.cloud/debian:bookworm RUN apt update && \