vbatts/maubot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Start working on management API implementation

Browse source
This commit is contained in:
Tulir Asokan 2018-10-28 01:31:03 +03:00
parent aefdcd9447
commit f2449e2eba
14 changed files with 379 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
example-config.yaml
View file

@ -5,24 +5,30 @@
# Postgres: postgres://username:password@hostname/dbname
database: sqlite:///maubot.db
# The directory where plugin databases should be stored.
plugin_db_directory: ./plugins
# If multiple directories have a plugin with the same name, the first directory is used.
plugin_directories:
- ./plugins
# The directory where uploaded new plugins should be stored.
upload: ./plugins
# The directories from which plugins should be loaded.
# Duplicate plugin IDs will be moved to the trash.
load:
- ./plugins
# The directory where old plugin versions and conflicting plugins should be moved.
# Set to "delete" to delete files immediately.
trash: ./trash
# The directory where plugin databases should be stored.
db: ./plugins
server:
# The IP and port to listen to.
hostname: 0.0.0.0
port: 29316
# The base management API path.
base_path: /_matrix/maubot
base_path: /_matrix/maubot/v1
# The base appservice API path. Use / for legacy appservice API and /_matrix/app/v1 for v1.
appservice_base_path: /_matrix/app/v1
# The shared secret to authorize users of the API.
# The shared secret to sign API access tokens.
# Set to "generate" to generate and save a new token at startup.
shared_secret: generate
unshared_secret: generate
admins:
- "@admin:example.com"

42
maubot/__main__.py
View file

@ -14,20 +14,23 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from sqlalchemy import orm
from time import time
import sqlalchemy as sql
import logging.config
import argparse
import asyncio
import signal
import copy
import sys
import signal
import os
from .config import Config
from .db import Base, init as init_db
from .server import MaubotServer
from .client import Client, init as init_client
from .loader import ZippedPluginLoader
from .plugin import PluginInstance, init as init_plugin_instance_class
from .loader import ZippedPluginLoader, MaubotZipImportError, IDConflictError
from .instance import PluginInstance, init as init_plugin_instance_class
from .management.api import init as init_management
from .__meta__ import __version__
parser = argparse.ArgumentParser(description="A plugin-based Matrix bot system.",
@ -57,9 +60,36 @@ loop = asyncio.get_event_loop()
init_db(db_session)
init_client(loop)
init_plugin_instance_class(config)
server = MaubotServer(config, loop)
ZippedPluginLoader.load_all(*config["plugin_directories"])
init_plugin_instance_class(db_session, config)
management_api = init_management(config, loop)
server = MaubotServer(config, management_api, loop)
trash_path = config["plugin_directories.trash"]
def trash(file_path: str, new_name: Optional[str] = None) -> None:
if trash_path == "delete":
os.remove(file_path)
else:
new_name = new_name or f"{int(time())}-{os.path.basename(file_path)}"
os.rename(file_path, os.path.abspath(os.path.join(trash_path, new_name)))
ZippedPluginLoader.log.debug("Preloading plugins...")
for directory in config["plugin_directories.load"]:
for file in os.listdir(directory):
if not file.endswith(".mbp"):
continue
path = os.path.abspath(os.path.join(directory, file))
try:
ZippedPluginLoader.get(path)
except MaubotZipImportError:
ZippedPluginLoader.log.exception(f"Failed to load plugin at {path}, trashing...")
trash(path)
except IDConflictError:
ZippedPluginLoader.log.warn(f"Duplicate plugin ID at {path}, trashing...")
trash(path)
plugins = PluginInstance.all()
for plugin in plugins:

7
maubot/client.py
View file

@ -13,7 +13,7 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from typing import Dict, List, Optional
from typing import Dict, List, Optional, Set, TYPE_CHECKING
from aiohttp import ClientSession
import asyncio
import logging
@ -24,6 +24,9 @@ from mautrix.types import (UserID, SyncToken, FilterID, ContentURI, StrippedStat
from .db import DBClient
from .matrix import MaubotMatrixClient
if TYPE_CHECKING:
from .instance import PluginInstance
log = logging.getLogger("maubot.client")
@ -32,6 +35,7 @@ class Client:
cache: Dict[UserID, 'Client'] = {}
http_client: ClientSession = None
references: Set['PluginInstance']
db_instance: DBClient
client: MaubotMatrixClient
@ -39,6 +43,7 @@ class Client:
self.db_instance = db_instance
self.cache[self.id] = self
self.log = log.getChild(self.id)
self.references = set()
self.client = MaubotMatrixClient(mxid=self.id, base_url=self.homeserver,
token=self.access_token, client_session=self.http_client,
log=self.log, loop=self.loop, store=self.db_instance)

2
maubot/config.py
View file

@ -16,7 +16,7 @@
import random
import string
from mautrix.util import BaseFileConfig, ConfigUpdateHelper
from mautrix.util.config import BaseFileConfig, ConfigUpdateHelper
class Config(BaseFileConfig):

33
maubot/plugin.py → maubot/instance.py
View file

@ -14,12 +14,13 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from typing import Dict, List, Optional
from sqlalchemy.orm import Session
from ruamel.yaml.comments import CommentedMap
from ruamel.yaml import YAML
import logging
import io
from mautrix.util import BaseProxyConfig, RecursiveDict
from mautrix.util.config import BaseProxyConfig, RecursiveDict
from mautrix.types import UserID
from .db import DBPlugin
@ -35,6 +36,7 @@ yaml.indent(4)
class PluginInstance:
db: Session = None
mb_config: Config = None
cache: Dict[str, 'PluginInstance'] = {}
plugin_directories: List[str] = []
@ -44,13 +46,24 @@ class PluginInstance:
client: Client
plugin: Plugin
config: BaseProxyConfig
running: bool
def __init__(self, db_instance: DBPlugin):
self.db_instance = db_instance
self.log = logging.getLogger(f"maubot.plugin.{self.id}")
self.config = None
self.running = False
self.cache[self.id] = self
def to_dict(self) -> dict:
return {
"id": self.id,
"type": self.type,
"enabled": self.enabled,
"running": self.running,
"primary_user": self.primary_user,
}
def load(self) -> None:
try:
self.loader = PluginLoader.find(self.type)
@ -63,6 +76,13 @@ class PluginInstance:
self.log.error(f"Failed to get client for user {self.primary_user}")
self.enabled = False
self.log.debug("Plugin instance dependencies loaded")
self.loader.references.add(self)
self.client.references.add(self)
def delete(self) -> None:
self.loader.references.remove(self)
self.db.delete(self.db_instance)
# TODO delete plugin db
def load_config(self) -> CommentedMap:
return yaml.load(self.db_instance.config)
@ -90,14 +110,14 @@ class PluginInstance:
self.save_config)
self.plugin = cls(self.client.client, self.id, self.log, self.config,
self.mb_config["plugin_db_directory"])
self.loader.references |= {self}
await self.plugin.start()
self.running = True
self.log.info(f"Started instance of {self.loader.id} v{self.loader.version} "
f"with user {self.client.id}")
async def stop(self) -> None:
self.log.debug("Stopping plugin instance...")
self.loader.references -= {self}
self.running = False
await self.plugin.stop()
self.plugin = None
@ -130,10 +150,6 @@ class PluginInstance:
def type(self) -> str:
return self.db_instance.type
@type.setter
def type(self, value: str) -> None:
self.db_instance.type = value
@property
def enabled(self) -> bool:
return self.db_instance.enabled
@ -153,5 +169,6 @@ class PluginInstance:
# endregion
def init(config: Config):
def init(db: Session, config: Config):
PluginInstance.db = db
PluginInstance.mb_config = config

2
maubot/loader/__init__.py
View file

@ -1,2 +1,2 @@
from .abc import PluginLoader, PluginClass
from .abc import PluginLoader, PluginClass, IDConflictError
from .zip import ZippedPluginLoader, MaubotZipImportError

19
maubot/loader/abc.py
View file

@ -15,11 +15,12 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from typing import TypeVar, Type, Dict, Set, TYPE_CHECKING
from abc import ABC, abstractmethod
import asyncio
from ..plugin_base import Plugin
if TYPE_CHECKING:
from ..plugin import PluginInstance
from ..instance import PluginInstance
PluginClass = TypeVar("PluginClass", bound=Plugin)
@ -42,6 +43,12 @@ class PluginLoader(ABC):
def find(cls, plugin_id: str) -> 'PluginLoader':
return cls.id_cache[plugin_id]
def to_dict(self) -> dict:
return {
"id": self.id,
"version": self.version,
}
@property
@abstractmethod
def source(self) -> str:
@ -51,6 +58,12 @@ class PluginLoader(ABC):
def read_file(self, path: str) -> bytes:
pass
async def stop_instances(self) -> None:
await asyncio.gather([instance.stop() for instance in self.references if instance.running])
async def start_instances(self) -> None:
await asyncio.gather([instance.start() for instance in self.references if instance.enabled])
@abstractmethod
def load(self) -> Type[PluginClass]:
pass
@ -62,3 +75,7 @@ class PluginLoader(ABC):
@abstractmethod
def unload(self) -> None:
pass
@abstractmethod
def delete(self) -> None:
pass

66
maubot/loader/zip.py
View file

@ -13,7 +13,7 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from typing import Dict, List, Type
from typing import Dict, List, Type, Tuple
from zipfile import ZipFile, BadZipFile
import configparser
import logging
@ -60,8 +60,15 @@ class ZippedPluginLoader(PluginLoader):
self.id_cache[self.id] = self
self.log.debug(f"Preloaded plugin {self.id} from {self.path}")
def to_dict(self) -> dict:
return {
**super().to_dict(),
"path": self.path
}
@classmethod
def get(cls, path: str) -> 'ZippedPluginLoader':
path = os.path.abspath(path)
try:
return cls.path_cache[path]
except KeyError:
@ -80,10 +87,11 @@ class ZippedPluginLoader(PluginLoader):
def read_file(self, path: str) -> bytes:
return self._file.read(path)
def _load_meta(self) -> None:
@staticmethod
def _open_meta(source) -> Tuple[ZipFile, configparser.ConfigParser]:
try:
self._file = ZipFile(self.path)
data = self._file.read("maubot.ini")
file = ZipFile(source)
data = file.read("maubot.ini")
except FileNotFoundError as e:
raise MaubotZipImportError("Maubot plugin not found") from e
except BadZipFile as e:
@ -92,7 +100,14 @@ class ZippedPluginLoader(PluginLoader):
raise MaubotZipImportError("File does not contain a maubot plugin definition") from e
config = configparser.ConfigParser()
try:
config.read_string(data.decode("utf-8"), source=f"{self.path}/maubot.ini")
config.read_string(data.decode("utf-8"))
except (configparser.Error, KeyError, IndexError, ValueError) as e:
raise MaubotZipImportError("Maubot plugin definition in file is invalid") from e
return file, config
@classmethod
def _read_meta(cls, config: configparser.ConfigParser) -> Tuple[str, str, List[str], str, str]:
try:
meta = config["maubot"]
meta_id = meta["ID"]
version = meta["Version"]
@ -103,10 +118,21 @@ class ZippedPluginLoader(PluginLoader):
main_module, main_class = main_class.split("/")[:2]
except (configparser.Error, KeyError, IndexError, ValueError) as e:
raise MaubotZipImportError("Maubot plugin definition in file is invalid") from e
if self.id and meta_id != self.id:
return meta_id, version, modules, main_class, main_module
@classmethod
def verify_meta(cls, source) -> Tuple[str, str]:
_, config = cls._open_meta(source)
meta = cls._read_meta(config)
return meta[0], meta[1]
def _load_meta(self) -> None:
file, config = self._open_meta(self.path)
meta = self._read_meta(config)
if self.id and meta[0] != self.id:
raise MaubotZipImportError("Maubot plugin ID changed during reload")
self.id, self.version, self.modules = meta_id, version, modules
self.main_class, self.main_module = main_class, main_module
self.id, self.version, self.modules, self.main_class, self.main_module = meta
self._file = file
def _get_importer(self, reset_cache: bool = False) -> zipimporter:
try:
@ -161,7 +187,7 @@ class ZippedPluginLoader(PluginLoader):
self._loaded = None
self.log.debug(f"Unloaded plugin {self.id} at {self.path}")
def destroy(self) -> None:
def delete(self) -> None:
self.unload()
try:
del self.path_cache[self.path]
@ -171,24 +197,12 @@ class ZippedPluginLoader(PluginLoader):
del self.id_cache[self.id]
except KeyError:
pass
self.id = None
self.path = None
self.version = None
self.modules = None
if self._importer:
self._importer.remove_cache()
self._importer = None
self._loaded = None
@classmethod
def load_all(cls, *args: str) -> None:
cls.log.debug("Preloading plugins...")
for directory in args:
for file in os.listdir(directory):
if not file.endswith(".mbp"):
continue
path = os.path.join(directory, file)
try:
ZippedPluginLoader.get(path)
except (MaubotZipImportError, IDConflictError):
cls.log.exception(f"Failed to load plugin at {path}")
os.remove(self.path)
self.id = None
self.path = None
self.version = None
self.modules = None

46
maubot/management/api/__init__.py
View file

@ -0,0 +1,46 @@
# maubot - A plugin-based Matrix bot system.
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from aiohttp import web
from asyncio import AbstractEventLoop
from mautrix.types import UserID
from mautrix.util.signed_token import sign_token, verify_token
from ...config import Config
routes = web.RouteTableDef()
config: Config = None
def is_valid_token(token: str) -> bool:
data = verify_token(config["server.unshared_secret"], token)
user_id = data.get("user_id", None)
return user_id is not None and user_id in config["admins"]
def create_token(user: UserID) -> str:
return sign_token(config["server.unshared_secret"], {
"user_id": user,
})
def init(cfg: Config, loop: AbstractEventLoop) -> web.Application:
global config
config = cfg
from .middleware import auth, error, log
app = web.Application(loop=loop, middlewares=[auth, log, error])
app.add_routes(routes)
return app

67
maubot/management/api/middleware.py Normal file
View file

@ -0,0 +1,67 @@
# maubot - A plugin-based Matrix bot system.
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from typing import Callable, Awaitable
from aiohttp import web
import logging
from .responses import ErrNoToken, ErrInvalidToken
from . import is_valid_token
Handler = Callable[[web.Request], Awaitable[web.Response]]
req_log = logging.getLogger("maubot.mgmt.request")
resp_log = logging.getLogger("maubot.mgmt.response")
@web.middleware
async def auth(request: web.Request, handler: Handler) -> web.Response:
token = request.headers.get("Authorization", "")
if not token or not token.startswith("Bearer "):
req_log.debug(f"Request missing auth: {request.remote} {request.method} {request.path}")
return ErrNoToken
if not is_valid_token(token[len("Bearer "):]):
req_log.debug(f"Request invalid auth: {request.remote} {request.method} {request.path}")
return ErrInvalidToken
return await handler(request)
@web.middleware
async def error(request: web.Request, handler: Handler) -> web.Response:
try:
return await handler(request)
except web.HTTPException as ex:
return web.json_response({
"error": f"Unhandled HTTP {ex.status}",
"errcode": f"unhandled_http_{ex.status}",
}, status=ex.status)
req_no = 0
def get_req_no():
global req_no
req_no += 1
return req_no
@web.middleware
async def log(request: web.Request, handler: Handler) -> web.Response:
local_req_no = get_req_no()
req_log.info(f"Request {local_req_no}: {request.remote} {request.method} {request.path}")
resp = await handler(request)
resp_log.info(f"Responded to {local_req_no} from {request.remote}: {resp}")
return resp

83
maubot/management/api/plugin.py Normal file
View file

@ -0,0 +1,83 @@
# maubot - A plugin-based Matrix bot system.
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from aiohttp import web
from io import BytesIO
import os.path
from ...loader import PluginLoader, ZippedPluginLoader, MaubotZipImportError
from .responses import ErrPluginNotFound, ErrPluginInUse, RespDeleted
from . import routes, config
def _plugin_to_dict(plugin: PluginLoader) -> dict:
return {
**plugin.to_dict(),
"instances": [instance.to_dict() for instance in plugin.references]
}
@routes.get("/plugins")
async def get_plugins(_) -> web.Response:
return web.json_response([_plugin_to_dict(plugin) for plugin in PluginLoader.id_cache.values()])
@routes.get("/plugin/{id}")
async def get_plugin(request: web.Request) -> web.Response:
plugin_id = request.match_info.get("id", None)
plugin = PluginLoader.id_cache.get(plugin_id, None)
if not plugin:
return ErrPluginNotFound
return web.json_response(_plugin_to_dict(plugin))
@routes.delete("/plugin/{id}")
async def delete_plugin(request: web.Request) -> web.Response:
plugin_id = request.match_info.get("id", None)
plugin = PluginLoader.id_cache.get(plugin_id, None)
if not plugin:
return ErrPluginNotFound
elif len(plugin.references) > 0:
return ErrPluginInUse
plugin.delete()
return RespDeleted
@routes.post("/plugins/upload")
async def upload_plugin(request: web.Request) -> web.Response:
content = await request.read()
file = BytesIO(content)
try:
pid, version = ZippedPluginLoader.verify_meta(file)
except MaubotZipImportError as e:
return web.json_response({
"error": str(e),
"errcode": "invalid_plugin",
}, status=web.HTTPBadRequest)
plugin = PluginLoader.id_cache.get(pid, None)
if not plugin:
path = os.path.join(config["plugin_directories.upload"], f"{pid}-{version}.mbp")
with open(path, "wb") as p:
p.write(content)
try:
ZippedPluginLoader.get(path)
except MaubotZipImportError as e:
trash(path)
return web.json_response({
"error": str(e),
"errcode": "invalid_plugin",
}, status=web.HTTPBadRequest)
else:
pass

38
maubot/management/api/responses.py Normal file
View file

@ -0,0 +1,38 @@
# maubot - A plugin-based Matrix bot system.
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from aiohttp import web
ErrNoToken = web.json_response({
"error": "Authorization token missing",
"errcode": "auth_token_missing",
}, status=web.HTTPUnauthorized)
ErrInvalidToken = web.json_response({
"error": "Invalid authorization token",
"errcode": "auth_token_invalid",
}, status=web.HTTPUnauthorized)
ErrPluginNotFound = web.json_response({
"error": "Plugin not found",
"errcode": "plugin_not_found",
}, status=web.HTTPNotFound)
ErrPluginInUse = web.json_response({
"error": "Plugin instances of this type still exist",
"errcode": "plugin_in_use",
})
RespDeleted = web.Response(status=204)

2
maubot/plugin_base.py
View file

@ -24,7 +24,7 @@ import sqlalchemy as sql
if TYPE_CHECKING:
from .client import MaubotMatrixClient
from .command_spec import CommandSpec
from mautrix.util import BaseProxyConfig
from mautrix.util.config import BaseProxyConfig
class Plugin(ABC):

4
maubot/server.py
View file

@ -23,13 +23,15 @@ from .__meta__ import __version__
class MaubotServer:
def __init__(self, config: Config, loop: asyncio.AbstractEventLoop):
def __init__(self, config: Config, management: web.Application,
loop: asyncio.AbstractEventLoop) -> None:
self.loop = loop or asyncio.get_event_loop()
self.app = web.Application(loop=self.loop)
self.config = config
path = PathBuilder(config["server.base_path"])
self.add_route(Method.GET, path.version, self.version)
self.app.add_subapp(config["server.base_path"], management)
as_path = PathBuilder(config["server.appservice_base_path"])
self.add_route(Method.PUT, as_path.transactions, self.handle_transaction)