Start working on management API implementation

example-config.yaml

@@ -5,24 +5,30 @@
 #   Postgres: postgres://username:password@hostname/dbname
 database: sqlite:///maubot.db
 
-# The directory where plugin databases should be stored.
-plugin_db_directory: ./plugins
-
-# If multiple directories have a plugin with the same name, the first directory is used.
 plugin_directories:
-- ./plugins
+    # The directory where uploaded new plugins should be stored.
+    upload: ./plugins
+    # The directories from which plugins should be loaded.
+    # Duplicate plugin IDs will be moved to the trash.
+    load:
+    - ./plugins
+    # The directory where old plugin versions and conflicting plugins should be moved.
+    # Set to "delete" to delete files immediately.
+    trash: ./trash
+    # The directory where plugin databases should be stored.
+    db: ./plugins
 
 server:
     # The IP and port to listen to.
     hostname: 0.0.0.0
     port: 29316
     # The base management API path.
-    base_path: /_matrix/maubot
+    base_path: /_matrix/maubot/v1
     # The base appservice API path. Use / for legacy appservice API and /_matrix/app/v1 for v1.
     appservice_base_path: /_matrix/app/v1
-    # The shared secret to authorize users of the API.
+    # The shared secret to sign API access tokens.
     # Set to "generate" to generate and save a new token at startup.
-    shared_secret: generate
+    unshared_secret: generate
 
 admins:
 - "@admin:example.com"

maubot/__main__.py

@@ -14,20 +14,23 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from sqlalchemy import orm
+from time import time
 import sqlalchemy as sql
 import logging.config
 import argparse
 import asyncio
+import signal
 import copy
 import sys
-import signal
+import os
 
 from .config import Config
 from .db import Base, init as init_db
 from .server import MaubotServer
 from .client import Client, init as init_client
-from .loader import ZippedPluginLoader
+from .loader import ZippedPluginLoader, MaubotZipImportError, IDConflictError
-from .plugin import PluginInstance, init as init_plugin_instance_class
+from .instance import PluginInstance, init as init_plugin_instance_class
+from .management.api import init as init_management
 from .__meta__ import __version__
 
 parser = argparse.ArgumentParser(description="A plugin-based Matrix bot system.",
@@ -57,9 +60,36 @@ loop = asyncio.get_event_loop()
 
 init_db(db_session)
 init_client(loop)
-init_plugin_instance_class(config)
+init_plugin_instance_class(db_session, config)
-server = MaubotServer(config, loop)
+management_api = init_management(config, loop)
-ZippedPluginLoader.load_all(*config["plugin_directories"])
+server = MaubotServer(config, management_api, loop)
+
+trash_path = config["plugin_directories.trash"]
+
+
+def trash(file_path: str, new_name: Optional[str] = None) -> None:
+    if trash_path == "delete":
+        os.remove(file_path)
+    else:
+        new_name = new_name or f"{int(time())}-{os.path.basename(file_path)}"
+        os.rename(file_path, os.path.abspath(os.path.join(trash_path, new_name)))
+
+
+ZippedPluginLoader.log.debug("Preloading plugins...")
+for directory in config["plugin_directories.load"]:
+    for file in os.listdir(directory):
+        if not file.endswith(".mbp"):
+            continue
+        path = os.path.abspath(os.path.join(directory, file))
+        try:
+            ZippedPluginLoader.get(path)
+        except MaubotZipImportError:
+            ZippedPluginLoader.log.exception(f"Failed to load plugin at {path}, trashing...")
+            trash(path)
+        except IDConflictError:
+            ZippedPluginLoader.log.warn(f"Duplicate plugin ID at {path}, trashing...")
+            trash(path)
+
 plugins = PluginInstance.all()
 
 for plugin in plugins:

maubot/client.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Set, TYPE_CHECKING
 from aiohttp import ClientSession
 import asyncio
 import logging
@@ -24,6 +24,9 @@ from mautrix.types import (UserID, SyncToken, FilterID, ContentURI, StrippedStat
 from .db import DBClient
 from .matrix import MaubotMatrixClient
 
+if TYPE_CHECKING:
+    from .instance import PluginInstance
+
 log = logging.getLogger("maubot.client")
 
 
@@ -32,6 +35,7 @@ class Client:
     cache: Dict[UserID, 'Client'] = {}
     http_client: ClientSession = None
 
+    references: Set['PluginInstance']
     db_instance: DBClient
     client: MaubotMatrixClient
 
@@ -39,6 +43,7 @@ class Client:
         self.db_instance = db_instance
         self.cache[self.id] = self
         self.log = log.getChild(self.id)
+        self.references = set()
         self.client = MaubotMatrixClient(mxid=self.id, base_url=self.homeserver,
                                          token=self.access_token, client_session=self.http_client,
                                          log=self.log, loop=self.loop, store=self.db_instance)

maubot/config.py

@@ -16,7 +16,7 @@
 import random
 import string
 
-from mautrix.util import BaseFileConfig, ConfigUpdateHelper
+from mautrix.util.config import BaseFileConfig, ConfigUpdateHelper
 
 
 class Config(BaseFileConfig):

maubot/instance.py

@@ -14,12 +14,13 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Dict, List, Optional
+from sqlalchemy.orm import Session
 from ruamel.yaml.comments import CommentedMap
 from ruamel.yaml import YAML
 import logging
 import io
 
-from mautrix.util import BaseProxyConfig, RecursiveDict
+from mautrix.util.config import BaseProxyConfig, RecursiveDict
 from mautrix.types import UserID
 
 from .db import DBPlugin
@@ -35,6 +36,7 @@ yaml.indent(4)
 
 
 class PluginInstance:
+    db: Session = None
     mb_config: Config = None
     cache: Dict[str, 'PluginInstance'] = {}
     plugin_directories: List[str] = []
@@ -44,13 +46,24 @@ class PluginInstance:
     client: Client
     plugin: Plugin
     config: BaseProxyConfig
+    running: bool
 
     def __init__(self, db_instance: DBPlugin):
         self.db_instance = db_instance
         self.log = logging.getLogger(f"maubot.plugin.{self.id}")
         self.config = None
+        self.running = False
         self.cache[self.id] = self
 
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "type": self.type,
+            "enabled": self.enabled,
+            "running": self.running,
+            "primary_user": self.primary_user,
+        }
+
     def load(self) -> None:
         try:
             self.loader = PluginLoader.find(self.type)
@@ -63,6 +76,13 @@ class PluginInstance:
             self.log.error(f"Failed to get client for user {self.primary_user}")
             self.enabled = False
         self.log.debug("Plugin instance dependencies loaded")
+        self.loader.references.add(self)
+        self.client.references.add(self)
+
+    def delete(self) -> None:
+        self.loader.references.remove(self)
+        self.db.delete(self.db_instance)
+        # TODO delete plugin db
 
     def load_config(self) -> CommentedMap:
         return yaml.load(self.db_instance.config)
@@ -90,14 +110,14 @@ class PluginInstance:
                                        self.save_config)
         self.plugin = cls(self.client.client, self.id, self.log, self.config,
                           self.mb_config["plugin_db_directory"])
-        self.loader.references |= {self}
         await self.plugin.start()
+        self.running = True
         self.log.info(f"Started instance of {self.loader.id} v{self.loader.version} "
                       f"with user {self.client.id}")
 
     async def stop(self) -> None:
         self.log.debug("Stopping plugin instance...")
-        self.loader.references -= {self}
+        self.running = False
         await self.plugin.stop()
         self.plugin = None
 
@@ -130,10 +150,6 @@ class PluginInstance:
     def type(self) -> str:
         return self.db_instance.type
 
-    @type.setter
-    def type(self, value: str) -> None:
-        self.db_instance.type = value
-
     @property
     def enabled(self) -> bool:
         return self.db_instance.enabled
@@ -153,5 +169,6 @@ class PluginInstance:
     # endregion
 
 
-def init(config: Config):
+def init(db: Session, config: Config):
+    PluginInstance.db = db
     PluginInstance.mb_config = config

maubot/loader/__init__.py

@@ -1,2 +1,2 @@
-from .abc import PluginLoader, PluginClass
+from .abc import PluginLoader, PluginClass, IDConflictError
 from .zip import ZippedPluginLoader, MaubotZipImportError

maubot/loader/abc.py

@@ -15,11 +15,12 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import TypeVar, Type, Dict, Set, TYPE_CHECKING
 from abc import ABC, abstractmethod
+import asyncio
 
 from ..plugin_base import Plugin
 
 if TYPE_CHECKING:
-    from ..plugin import PluginInstance
+    from ..instance import PluginInstance
 
 PluginClass = TypeVar("PluginClass", bound=Plugin)
 
@@ -42,6 +43,12 @@ class PluginLoader(ABC):
     def find(cls, plugin_id: str) -> 'PluginLoader':
         return cls.id_cache[plugin_id]
 
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "version": self.version,
+        }
+
     @property
     @abstractmethod
     def source(self) -> str:
@@ -51,6 +58,12 @@ class PluginLoader(ABC):
     def read_file(self, path: str) -> bytes:
         pass
 
+    async def stop_instances(self) -> None:
+        await asyncio.gather([instance.stop() for instance in self.references if instance.running])
+
+    async def start_instances(self) -> None:
+        await asyncio.gather([instance.start() for instance in self.references if instance.enabled])
+
     @abstractmethod
     def load(self) -> Type[PluginClass]:
         pass
@@ -62,3 +75,7 @@ class PluginLoader(ABC):
     @abstractmethod
     def unload(self) -> None:
         pass
+
+    @abstractmethod
+    def delete(self) -> None:
+        pass

maubot/loader/zip.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Dict, List, Type
+from typing import Dict, List, Type, Tuple
 from zipfile import ZipFile, BadZipFile
 import configparser
 import logging
@@ -60,8 +60,15 @@ class ZippedPluginLoader(PluginLoader):
         self.id_cache[self.id] = self
         self.log.debug(f"Preloaded plugin {self.id} from {self.path}")
 
+    def to_dict(self) -> dict:
+        return {
+            **super().to_dict(),
+            "path": self.path
+        }
+
     @classmethod
     def get(cls, path: str) -> 'ZippedPluginLoader':
+        path = os.path.abspath(path)
         try:
             return cls.path_cache[path]
         except KeyError:
@@ -80,10 +87,11 @@ class ZippedPluginLoader(PluginLoader):
     def read_file(self, path: str) -> bytes:
         return self._file.read(path)
 
-    def _load_meta(self) -> None:
+    @staticmethod
+    def _open_meta(source) -> Tuple[ZipFile, configparser.ConfigParser]:
         try:
-            self._file = ZipFile(self.path)
+            file = ZipFile(source)
-            data = self._file.read("maubot.ini")
+            data = file.read("maubot.ini")
         except FileNotFoundError as e:
             raise MaubotZipImportError("Maubot plugin not found") from e
         except BadZipFile as e:
@@ -92,7 +100,14 @@ class ZippedPluginLoader(PluginLoader):
             raise MaubotZipImportError("File does not contain a maubot plugin definition") from e
         config = configparser.ConfigParser()
         try:
-            config.read_string(data.decode("utf-8"), source=f"{self.path}/maubot.ini")
+            config.read_string(data.decode("utf-8"))
+        except (configparser.Error, KeyError, IndexError, ValueError) as e:
+            raise MaubotZipImportError("Maubot plugin definition in file is invalid") from e
+        return file, config
+
+    @classmethod
+    def _read_meta(cls, config: configparser.ConfigParser) -> Tuple[str, str, List[str], str, str]:
+        try:
             meta = config["maubot"]
             meta_id = meta["ID"]
             version = meta["Version"]
@@ -103,10 +118,21 @@ class ZippedPluginLoader(PluginLoader):
                 main_module, main_class = main_class.split("/")[:2]
         except (configparser.Error, KeyError, IndexError, ValueError) as e:
             raise MaubotZipImportError("Maubot plugin definition in file is invalid") from e
-        if self.id and meta_id != self.id:
+        return meta_id, version, modules, main_class, main_module
+
+    @classmethod
+    def verify_meta(cls, source) -> Tuple[str, str]:
+        _, config = cls._open_meta(source)
+        meta = cls._read_meta(config)
+        return meta[0], meta[1]
+
+    def _load_meta(self) -> None:
+        file, config = self._open_meta(self.path)
+        meta = self._read_meta(config)
+        if self.id and meta[0] != self.id:
             raise MaubotZipImportError("Maubot plugin ID changed during reload")
-        self.id, self.version, self.modules = meta_id, version, modules
+        self.id, self.version, self.modules, self.main_class, self.main_module = meta
-        self.main_class, self.main_module = main_class, main_module
+        self._file = file
 
     def _get_importer(self, reset_cache: bool = False) -> zipimporter:
         try:
@@ -161,7 +187,7 @@ class ZippedPluginLoader(PluginLoader):
         self._loaded = None
         self.log.debug(f"Unloaded plugin {self.id} at {self.path}")
 
-    def destroy(self) -> None:
+    def delete(self) -> None:
         self.unload()
         try:
             del self.path_cache[self.path]
@@ -171,24 +197,12 @@ class ZippedPluginLoader(PluginLoader):
             del self.id_cache[self.id]
         except KeyError:
             pass
-        self.id = None
-        self.path = None
-        self.version = None
-        self.modules = None
         if self._importer:
             self._importer.remove_cache()
             self._importer = None
         self._loaded = None
-
+        os.remove(self.path)
-    @classmethod
+        self.id = None
-    def load_all(cls, *args: str) -> None:
+        self.path = None
-        cls.log.debug("Preloading plugins...")
+        self.version = None
-        for directory in args:
+        self.modules = None
-            for file in os.listdir(directory):
-                if not file.endswith(".mbp"):
-                    continue
-                path = os.path.join(directory, file)
-                try:
-                    ZippedPluginLoader.get(path)
-                except (MaubotZipImportError, IDConflictError):
-                    cls.log.exception(f"Failed to load plugin at {path}")

maubot/management/api/__init__.py

@@ -0,0 +1,46 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from aiohttp import web
+from asyncio import AbstractEventLoop
+
+from mautrix.types import UserID
+from mautrix.util.signed_token import sign_token, verify_token
+
+from ...config import Config
+
+routes = web.RouteTableDef()
+config: Config = None
+
+
+def is_valid_token(token: str) -> bool:
+    data = verify_token(config["server.unshared_secret"], token)
+    user_id = data.get("user_id", None)
+    return user_id is not None and user_id in config["admins"]
+
+
+def create_token(user: UserID) -> str:
+    return sign_token(config["server.unshared_secret"], {
+        "user_id": user,
+    })
+
+
+def init(cfg: Config, loop: AbstractEventLoop) -> web.Application:
+    global config
+    config = cfg
+    from .middleware import auth, error, log
+    app = web.Application(loop=loop, middlewares=[auth, log, error])
+    app.add_routes(routes)
+    return app

maubot/management/api/middleware.py

@@ -0,0 +1,67 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Callable, Awaitable
+from aiohttp import web
+import logging
+
+from .responses import ErrNoToken, ErrInvalidToken
+from . import is_valid_token
+
+Handler = Callable[[web.Request], Awaitable[web.Response]]
+
+req_log = logging.getLogger("maubot.mgmt.request")
+resp_log = logging.getLogger("maubot.mgmt.response")
+
+
+@web.middleware
+async def auth(request: web.Request, handler: Handler) -> web.Response:
+    token = request.headers.get("Authorization", "")
+    if not token or not token.startswith("Bearer "):
+        req_log.debug(f"Request missing auth: {request.remote} {request.method} {request.path}")
+        return ErrNoToken
+    if not is_valid_token(token[len("Bearer "):]):
+        req_log.debug(f"Request invalid auth: {request.remote} {request.method} {request.path}")
+        return ErrInvalidToken
+    return await handler(request)
+
+
+@web.middleware
+async def error(request: web.Request, handler: Handler) -> web.Response:
+    try:
+        return await handler(request)
+    except web.HTTPException as ex:
+        return web.json_response({
+            "error": f"Unhandled HTTP {ex.status}",
+            "errcode": f"unhandled_http_{ex.status}",
+        }, status=ex.status)
+
+
+req_no = 0
+
+
+def get_req_no():
+    global req_no
+    req_no += 1
+    return req_no
+
+
+@web.middleware
+async def log(request: web.Request, handler: Handler) -> web.Response:
+    local_req_no = get_req_no()
+    req_log.info(f"Request {local_req_no}: {request.remote} {request.method} {request.path}")
+    resp = await handler(request)
+    resp_log.info(f"Responded to {local_req_no} from {request.remote}: {resp}")
+    return resp

maubot/management/api/plugin.py

@@ -0,0 +1,83 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from aiohttp import web
+from io import BytesIO
+import os.path
+
+from ...loader import PluginLoader, ZippedPluginLoader, MaubotZipImportError
+from .responses import ErrPluginNotFound, ErrPluginInUse, RespDeleted
+from . import routes, config
+
+
+def _plugin_to_dict(plugin: PluginLoader) -> dict:
+    return {
+        **plugin.to_dict(),
+        "instances": [instance.to_dict() for instance in plugin.references]
+    }
+
+
+@routes.get("/plugins")
+async def get_plugins(_) -> web.Response:
+    return web.json_response([_plugin_to_dict(plugin) for plugin in PluginLoader.id_cache.values()])
+
+
+@routes.get("/plugin/{id}")
+async def get_plugin(request: web.Request) -> web.Response:
+    plugin_id = request.match_info.get("id", None)
+    plugin = PluginLoader.id_cache.get(plugin_id, None)
+    if not plugin:
+        return ErrPluginNotFound
+    return web.json_response(_plugin_to_dict(plugin))
+
+
+@routes.delete("/plugin/{id}")
+async def delete_plugin(request: web.Request) -> web.Response:
+    plugin_id = request.match_info.get("id", None)
+    plugin = PluginLoader.id_cache.get(plugin_id, None)
+    if not plugin:
+        return ErrPluginNotFound
+    elif len(plugin.references) > 0:
+        return ErrPluginInUse
+    plugin.delete()
+    return RespDeleted
+
+
+@routes.post("/plugins/upload")
+async def upload_plugin(request: web.Request) -> web.Response:
+    content = await request.read()
+    file = BytesIO(content)
+    try:
+        pid, version = ZippedPluginLoader.verify_meta(file)
+    except MaubotZipImportError as e:
+        return web.json_response({
+            "error": str(e),
+            "errcode": "invalid_plugin",
+        }, status=web.HTTPBadRequest)
+    plugin = PluginLoader.id_cache.get(pid, None)
+    if not plugin:
+        path = os.path.join(config["plugin_directories.upload"], f"{pid}-{version}.mbp")
+        with open(path, "wb") as p:
+            p.write(content)
+        try:
+            ZippedPluginLoader.get(path)
+        except MaubotZipImportError as e:
+            trash(path)
+            return web.json_response({
+                "error": str(e),
+                "errcode": "invalid_plugin",
+            }, status=web.HTTPBadRequest)
+    else:
+        pass

maubot/management/api/responses.py

@@ -0,0 +1,38 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from aiohttp import web
+
+ErrNoToken = web.json_response({
+    "error": "Authorization token missing",
+    "errcode": "auth_token_missing",
+}, status=web.HTTPUnauthorized)
+
+ErrInvalidToken = web.json_response({
+    "error": "Invalid authorization token",
+    "errcode": "auth_token_invalid",
+}, status=web.HTTPUnauthorized)
+
+ErrPluginNotFound = web.json_response({
+    "error": "Plugin not found",
+    "errcode": "plugin_not_found",
+}, status=web.HTTPNotFound)
+
+ErrPluginInUse = web.json_response({
+    "error": "Plugin instances of this type still exist",
+    "errcode": "plugin_in_use",
+})
+
+RespDeleted = web.Response(status=204)

maubot/plugin_base.py

@@ -24,7 +24,7 @@ import sqlalchemy as sql
 if TYPE_CHECKING:
     from .client import MaubotMatrixClient
     from .command_spec import CommandSpec
-    from mautrix.util import BaseProxyConfig
+    from mautrix.util.config import BaseProxyConfig
 
 
 class Plugin(ABC):

maubot/server.py

@@ -23,13 +23,15 @@ from .__meta__ import __version__
 
 
 class MaubotServer:
-    def __init__(self, config: Config, loop: asyncio.AbstractEventLoop):
+    def __init__(self, config: Config, management: web.Application,
+                 loop: asyncio.AbstractEventLoop) -> None:
         self.loop = loop or asyncio.get_event_loop()
         self.app = web.Application(loop=self.loop)
         self.config = config
 
         path = PathBuilder(config["server.base_path"])
         self.add_route(Method.GET, path.version, self.version)
+        self.app.add_subapp(config["server.base_path"], management)
 
         as_path = PathBuilder(config["server.appservice_base_path"])
         self.add_route(Method.PUT, as_path.transactions, self.handle_transaction)