From 7570f3c03dc6b398ae52ac6d2364a9c711831892 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 4 Aug 2019 10:06:33 +0200 Subject: [PATCH] Cleanup little-boxes stuff --- app.py | 13 +++--- blueprints/api.py | 4 +- blueprints/tasks.py | 85 +++++++++------------------------------ core/activitypub.py | 37 +++-------------- core/db.py | 8 ++++ core/shared.py | 22 ++++++++-- utils/template_filters.py | 2 +- 7 files changed, 61 insertions(+), 110 deletions(-) diff --git a/app.py b/app.py index ee357ca..6f5fd04 100644 --- a/app.py +++ b/app.py @@ -55,6 +55,7 @@ from core.shared import MY_PERSON from core.shared import _add_answers_to_question from core.shared import _build_thread from core.shared import _get_ip +from core.shared import activity_url from core.shared import back from core.shared import csrf from core.shared import login_required @@ -410,7 +411,7 @@ def note_by_id(note_id): return redirect(url_for("outbox_activity", item_id=note_id)) data = DB.activities.find_one( - {"box": Box.OUTBOX.value, "remote_id": back.activity_url(note_id)} + {"box": Box.OUTBOX.value, "remote_id": activity_url(note_id)} ) if not data: abort(404) @@ -548,7 +549,7 @@ def outbox_detail(item_id): doc = DB.activities.find_one( { "box": Box.OUTBOX.value, - "remote_id": back.activity_url(item_id), + "remote_id": activity_url(item_id), "meta.public": True, } ) @@ -566,7 +567,7 @@ def outbox_detail(item_id): @app.route("/outbox//activity") def outbox_activity(item_id): data = find_one_activity( - {**in_outbox(), **by_remote_id(back.activity_url(item_id)), **is_public()} + {**in_outbox(), **by_remote_id(activity_url(item_id)), **is_public()} ) if not data: abort(404) @@ -590,7 +591,7 @@ def outbox_activity_replies(item_id): data = DB.activities.find_one( { "box": Box.OUTBOX.value, - "remote_id": back.activity_url(item_id), + "remote_id": activity_url(item_id), "meta.deleted": False, "meta.public": True, } @@ -627,7 +628,7 @@ def outbox_activity_likes(item_id): data = DB.activities.find_one( { "box": Box.OUTBOX.value, - "remote_id": back.activity_url(item_id), + "remote_id": activity_url(item_id), "meta.deleted": False, "meta.public": True, } @@ -666,7 +667,7 @@ def outbox_activity_shares(item_id): data = DB.activities.find_one( { "box": Box.OUTBOX.value, - "remote_id": back.activity_url(item_id), + "remote_id": activity_url(item_id), "meta.deleted": False, } ) diff --git a/blueprints/api.py b/blueprints/api.py index a1cd1c4..4a29aa9 100644 --- a/blueprints/api.py +++ b/blueprints/api.py @@ -38,7 +38,7 @@ from core.meta import MetaKey from core.meta import _meta from core.shared import MY_PERSON from core.shared import _Response -from core.shared import back +from core.shared import activity_url from core.shared import csrf from core.shared import login_required from core.shared import post_to_outbox @@ -291,7 +291,7 @@ def api_undo() -> _Response: doc = DB.activities.find_one( { "box": Box.OUTBOX.value, - "$or": [{"remote_id": back.activity_url(oid)}, {"remote_id": oid}], + "$or": [{"remote_id": activity_url(oid)}, {"remote_id": oid}], } ) if not doc: diff --git a/blueprints/tasks.py b/blueprints/tasks.py index f10892b..0927e5f 100644 --- a/blueprints/tasks.py +++ b/blueprints/tasks.py @@ -14,13 +14,15 @@ from little_boxes.httpsig import HTTPSigAuth from requests.exceptions import HTTPError import config +from core.shared import _Response from config import DB -from core import activitypub from core import gc from core.activitypub import Box +from core.inbox import process_inbox from core.meta import MetaKey from core.meta import _meta from core.notifications import set_inbox_flags +from core.outbox import process_outbox from core.shared import MY_PERSON from core.shared import _add_answers_to_question from core.shared import back @@ -43,7 +45,7 @@ class TaskError(Exception): @blueprint.route("/task/update_question", methods=["POST"]) -def task_update_question(): +def task_update_question() -> _Response: """Sends an Update.""" task = p.parse(flask.request) app.logger.info(f"task={task!r}") @@ -84,7 +86,7 @@ def task_update_question(): @blueprint.route("/task/fetch_og_meta", methods=["POST"]) -def task_fetch_og_meta(): +def task_fetch_og_meta() -> _Response: task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload @@ -123,7 +125,7 @@ def task_fetch_og_meta(): @blueprint.route("/task/cache_object", methods=["POST"]) -def task_cache_object(): +def task_cache_object() -> _Response: task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload @@ -136,7 +138,8 @@ def task_cache_object(): { "$set": { "meta.object": obj.to_dict(embed=True), - "meta.object_actor": activitypub._actor_to_meta(obj.get_actor()), + # FIXME(tsileo): set object actor only if different from actor? + "meta.object_actor": obj.get_actor().to_dict(embed=True), } }, ) @@ -151,7 +154,7 @@ def task_cache_object(): @blueprint.route("/task/finish_post_to_outbox", methods=["POST"]) # noqa:C901 -def task_finish_post_to_outbox(): +def task_finish_post_to_outbox() -> _Response: task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload @@ -161,24 +164,7 @@ def task_finish_post_to_outbox(): recipients = activity.recipients() - if activity.has_type(ap.ActivityType.DELETE): - back.outbox_delete(MY_PERSON, activity) - elif activity.has_type(ap.ActivityType.UPDATE): - back.outbox_update(MY_PERSON, activity) - elif activity.has_type(ap.ActivityType.CREATE): - back.outbox_create(MY_PERSON, activity) - elif activity.has_type(ap.ActivityType.ANNOUNCE): - back.outbox_announce(MY_PERSON, activity) - elif activity.has_type(ap.ActivityType.LIKE): - back.outbox_like(MY_PERSON, activity) - elif activity.has_type(ap.ActivityType.UNDO): - obj = activity.get_object() - if obj.has_type(ap.ActivityType.LIKE): - back.outbox_undo_like(MY_PERSON, obj) - elif obj.has_type(ap.ActivityType.ANNOUNCE): - back.outbox_undo_announce(MY_PERSON, obj) - elif obj.has_type(ap.ActivityType.FOLLOW): - back.undo_new_following(MY_PERSON, obj) + process_outbox(activity, {}) app.logger.info(f"recipients={recipients}") activity = ap.clean_activity(activity.to_dict()) @@ -197,7 +183,7 @@ def task_finish_post_to_outbox(): @blueprint.route("/task/finish_post_to_inbox", methods=["POST"]) # noqa: C901 -def task_finish_post_to_inbox(): +def task_finish_post_to_inbox() -> _Response: task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload @@ -205,39 +191,8 @@ def task_finish_post_to_inbox(): activity = ap.fetch_remote_activity(iri) app.logger.info(f"activity={activity!r}") - if activity.has_type(ap.ActivityType.DELETE): - back.inbox_delete(MY_PERSON, activity) - elif activity.has_type(ap.ActivityType.UPDATE): - back.inbox_update(MY_PERSON, activity) - elif activity.has_type(ap.ActivityType.CREATE): - back.inbox_create(MY_PERSON, activity) - elif activity.has_type(ap.ActivityType.ANNOUNCE): - back.inbox_announce(MY_PERSON, activity) - elif activity.has_type(ap.ActivityType.LIKE): - back.inbox_like(MY_PERSON, activity) - elif activity.has_type(ap.ActivityType.FOLLOW): - # Reply to a Follow with an Accept - actor_id = activity.get_actor().id - accept = ap.Accept( - actor=config.ID, - object={ - "type": "Follow", - "id": activity.id, - "object": activity.get_object_id(), - "actor": actor_id, - }, - to=[actor_id], - published=now(), - ) - post_to_outbox(accept) - elif activity.has_type(ap.ActivityType.UNDO): - obj = activity.get_object() - if obj.has_type(ap.ActivityType.LIKE): - back.inbox_undo_like(MY_PERSON, obj) - elif obj.has_type(ap.ActivityType.ANNOUNCE): - back.inbox_undo_announce(MY_PERSON, obj) - elif obj.has_type(ap.ActivityType.FOLLOW): - back.undo_new_follower(MY_PERSON, obj) + process_inbox(activity, {}) + except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): app.logger.exception(f"no retry") except Exception as err: @@ -248,7 +203,7 @@ def task_finish_post_to_inbox(): @blueprint.route("/task/cache_attachments", methods=["POST"]) -def task_cache_attachments(): +def task_cache_attachments() -> _Response: task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload @@ -278,7 +233,7 @@ def task_cache_attachments(): @blueprint.route("/task/cache_actor", methods=["POST"]) -def task_cache_actor() -> str: +def task_cache_actor() -> _Response: task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload["iri"] @@ -329,7 +284,7 @@ def task_cache_actor() -> str: @blueprint.route("/task/forward_activity", methods=["POST"]) -def task_forward_activity(): +def task_forward_activity() -> _Response: task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload @@ -350,7 +305,7 @@ def task_forward_activity(): @blueprint.route("/task/post_to_remote_inbox", methods=["POST"]) -def task_post_to_remote_inbox(): +def task_post_to_remote_inbox() -> _Response: """Post an activity to a remote inbox.""" task = p.parse(flask.request) app.logger.info(f"task={task!r}") @@ -394,7 +349,7 @@ def task_post_to_remote_inbox(): @blueprint.route("/task/fetch_remote_question", methods=["POST"]) -def task_fetch_remote_question(): +def task_fetch_remote_question() -> _Response: """Fetch a remote question for implementation that does not send Update.""" task = p.parse(flask.request) app.logger.info(f"task={task!r}") @@ -454,7 +409,7 @@ def task_fetch_remote_question(): @blueprint.route("/task/cleanup", methods=["POST"]) -def task_cleanup(): +def task_cleanup() -> _Response: task = p.parse(flask.request) app.logger.info(f"task={task!r}") gc.perform() @@ -462,7 +417,7 @@ def task_cleanup(): @blueprint.route("/task/process_new_activity", methods=["POST"]) # noqa:c901 -def task_process_new_activity(): +def task_process_new_activity() -> _Response: """Process an activity received in the inbox""" task = p.parse(flask.request) app.logger.info(f"task={task!r}") diff --git a/core/activitypub.py b/core/activitypub.py index 77b55f6..b5cf1d9 100644 --- a/core/activitypub.py +++ b/core/activitypub.py @@ -1,5 +1,4 @@ import hashlib -import json import logging import os from datetime import datetime @@ -33,6 +32,8 @@ from core.tasks import Tasks logger = logging.getLogger(__name__) +_NewMeta = Dict[str, Any] + ACTORS_CACHE = LRUCache(maxsize=256) MY_PERSON = ap.Person(**ME) @@ -98,6 +99,9 @@ def _is_local_reply(create: ap.Create) -> bool: class MicroblogPubBackend(Backend): """Implements a Little Boxes backend, backed by MongoDB.""" + def base_url(self) -> str: + return BASE_URL + def debug_mode(self) -> bool: return strtobool(os.getenv("MICROBLOGPUB_DEBUG", "false")) @@ -108,18 +112,6 @@ class MicroblogPubBackend(Backend): def extra_inboxes(self) -> List[str]: return EXTRA_INBOXES - def base_url(self) -> str: - """Base URL config.""" - return BASE_URL - - def activity_url(self, obj_id): - """URL for activity link.""" - return f"{BASE_URL}/outbox/{obj_id}" - - def note_url(self, obj_id): - """URL for activity link.""" - return f"{BASE_URL}/note/{obj_id}" - def save(self, box: Box, activity: ap.BaseActivity) -> None: """Custom helper for saving an activity to the DB.""" visibility = ap.get_visibility(activity) @@ -695,25 +687,6 @@ class MicroblogPubBackend(Backend): {"$set": {"meta.thread_root_parent": root_reply}}, ) - def post_to_outbox(self, activity: ap.BaseActivity) -> None: - if activity.has_type(ap.CREATE_TYPES): - activity = activity.build_create() - - self.save(Box.OUTBOX, activity) - - # Assign create a random ID - obj_id = self.random_object_id() - activity.set_id(self.activity_url(obj_id), obj_id) - - recipients = activity.recipients() - logger.info(f"recipients={recipients}") - activity = ap.clean_activity(activity.to_dict()) - - payload = json.dumps(activity) - for recp in recipients: - logger.debug(f"posting to {recp}") - self.post_to_remote_inbox(self.get_actor(), payload, recp) - def gen_feed(): fg = FeedGenerator() diff --git a/core/db.py b/core/db.py index 607170c..59b5429 100644 --- a/core/db.py +++ b/core/db.py @@ -17,3 +17,11 @@ class CollectionName(Enum): def find_one_activity(q: _Q) -> _Doc: return DB[CollectionName.ACTIVITIES.value].find_one(q) + + +def update_one_activity(q: _Q, update: _Q) -> None: + DB[CollectionName.ACTIVITIES.value].update_one(q, update) + + +def update_many_activities(q: _Q, update: _Q) -> None: + DB[CollectionName.ACTIVITIES.value].update_many(q, update) diff --git a/core/shared.py b/core/shared.py index 367b179..dfda989 100644 --- a/core/shared.py +++ b/core/shared.py @@ -5,6 +5,7 @@ from functools import wraps from typing import Any from typing import Dict from typing import Union +from urllib.parse import urljoin import flask import werkzeug @@ -19,6 +20,7 @@ from little_boxes import activitypub as ap from little_boxes.activitypub import format_datetime from poussetaches import PousseTaches +from config import BASE_URL from config import DB from config import ME from core import activitypub @@ -26,7 +28,8 @@ from core.activitypub import _answer_key from core.meta import Box from core.tasks import Tasks -_Response = Union[flask.Response, werkzeug.wrappers.Response, str] +# _Response = Union[flask.Response, werkzeug.wrappers.Response, str, Any] +_Response = Any p = PousseTaches( os.getenv("MICROBLOGPUB_POUSSETACHES_HOST", "http://localhost:7991"), @@ -69,7 +72,7 @@ def login_required(f): @wraps(f) def decorated_function(*args, **kwargs): if not session.get("logged_in"): - return redirect(url_for("admin_login", next=request.url)) + return redirect(url_for("admin.admin_login", next=request.url)) return f(*args, **kwargs) return decorated_function @@ -92,14 +95,25 @@ def _get_ip(): return ip, geoip +def activity_url(item_id: str) -> str: + return urljoin(BASE_URL, url_for("outbox_detail", item_id=item_id)) + + def post_to_outbox(activity: ap.BaseActivity) -> str: if activity.has_type(ap.CREATE_TYPES): activity = activity.build_create() # Assign create a random ID obj_id = back.random_object_id() - - activity.set_id(back.activity_url(obj_id), obj_id) + uri = activity_url(obj_id) + activity._data["id"] = uri + if activity.has_type(ap.ActivityType.CREATE): + activity._data["object"]["id"] = urljoin( + BASE_URL, url_for("outbox_activity", node_id=obj_id) + ) + activity._data["object"]["url"] = urljoin( + BASE_URL, url_for("note_by_id", node_id=obj_id) + ) back.save(Box.OUTBOX, activity) Tasks.cache_actor(activity.id) diff --git a/utils/template_filters.py b/utils/template_filters.py index 8dedfe4..1f23b8f 100644 --- a/utils/template_filters.py +++ b/utils/template_filters.py @@ -17,10 +17,10 @@ from little_boxes.activitypub import _to_list from little_boxes.errors import ActivityGoneError from little_boxes.errors import ActivityNotFoundError -from core.activitypub import _answer_key from config import EMOJI_TPL from config import ID from config import MEDIA_CACHE +from core.activitypub import _answer_key from utils import parse_datetime from utils.media import Kind