diff --git a/app.py b/app.py index 5a1934c..ee2fc85 100644 --- a/app.py +++ b/app.py @@ -3,8 +3,6 @@ import logging import os import traceback from datetime import datetime -from typing import Any -from typing import Dict from urllib.parse import urlparse from bson.objectid import ObjectId @@ -20,6 +18,7 @@ from flask import url_for from itsdangerous import BadSignature from little_boxes import activitypub as ap from little_boxes.activitypub import ActivityType +from little_boxes.activitypub import activity_from_doc from little_boxes.activitypub import clean_activity from little_boxes.activitypub import get_backend from little_boxes.errors import ActivityGoneError @@ -43,10 +42,11 @@ from config import ME from config import MEDIA_CACHE from config import VERSION from core import activitypub +from core import feed from core.activitypub import activity_url -from core.activitypub import embed_collection from core.activitypub import post_to_inbox from core.activitypub import post_to_outbox +from core.activitypub import remove_context from core.db import find_one_activity from core.meta import Box from core.meta import MetaKey @@ -55,7 +55,6 @@ from core.meta import by_remote_id from core.meta import in_outbox from core.meta import is_public from core.shared import MY_PERSON -from core.shared import _add_answers_to_question from core.shared import _build_thread from core.shared import _get_ip from core.shared import csrf @@ -334,11 +333,6 @@ def u2f_register(): ####### # Activity pub routes -@app.route("/drop_cache") -@login_required -def drop_cache(): - DB.actors.drop() - return "Done" @app.route("/") @@ -468,44 +462,6 @@ def note_by_id(note_id): ) -def add_extra_collection(raw_doc: Dict[str, Any]) -> Dict[str, Any]: - if raw_doc["activity"]["type"] != ActivityType.CREATE.value: - return raw_doc - - raw_doc["activity"]["object"]["replies"] = embed_collection( - raw_doc.get("meta", {}).get("count_direct_reply", 0), - f'{raw_doc["remote_id"]}/replies', - ) - - raw_doc["activity"]["object"]["likes"] = embed_collection( - raw_doc.get("meta", {}).get("count_like", 0), f'{raw_doc["remote_id"]}/likes' - ) - - raw_doc["activity"]["object"]["shares"] = embed_collection( - raw_doc.get("meta", {}).get("count_boost", 0), f'{raw_doc["remote_id"]}/shares' - ) - - return raw_doc - - -def remove_context(activity: Dict[str, Any]) -> Dict[str, Any]: - if "@context" in activity: - del activity["@context"] - return activity - - -def activity_from_doc(raw_doc: Dict[str, Any], embed: bool = False) -> Dict[str, Any]: - raw_doc = add_extra_collection(raw_doc) - activity = clean_activity(raw_doc["activity"]) - - # Handle Questions - # TODO(tsileo): what about object embedded by ID/URL? - _add_answers_to_question(raw_doc) - if embed: - return remove_context(activity) - return activity - - @app.route("/outbox", methods=["GET", "POST"]) def outbox(): if request.method == "GET": @@ -986,7 +942,7 @@ def liked(): @app.route("/feed.json") def json_feed(): return Response( - response=json.dumps(activitypub.json_feed("/feed.json")), + response=json.dumps(feed.json_feed("/feed.json")), headers={"Content-Type": "application/json"}, ) @@ -994,7 +950,7 @@ def json_feed(): @app.route("/feed.atom") def atom_feed(): return Response( - response=activitypub.gen_feed().atom_str(), + response=feed.gen_feed().atom_str(), headers={"Content-Type": "application/atom+xml"}, ) @@ -1002,6 +958,6 @@ def atom_feed(): @app.route("/feed.rss") def rss_feed(): return Response( - response=activitypub.gen_feed().rss_str(), + response=feed.gen_feed().rss_str(), headers={"Content-Type": "application/rss+xml"}, ) diff --git a/core/activitypub.py b/core/activitypub.py index a5cb86f..a089ad2 100644 --- a/core/activitypub.py +++ b/core/activitypub.py @@ -11,12 +11,11 @@ from urllib.parse import urlparse from bson.objectid import ObjectId from cachetools import LRUCache -from feedgen.feed import FeedGenerator from flask import url_for -from html2text import html2text from little_boxes import activitypub as ap from little_boxes import strtobool from little_boxes.activitypub import _to_list +from little_boxes.activitypub import clean_activity from little_boxes.backend import Backend from little_boxes.errors import ActivityGoneError @@ -26,8 +25,8 @@ from config import EXTRA_INBOXES from config import ID from config import ME from config import USER_AGENT -from config import USERNAME from core.meta import Box +from core.shared import _add_answers_to_question from core.tasks import Tasks logger = logging.getLogger(__name__) @@ -457,118 +456,6 @@ class MicroblogPubBackend(Backend): ) -def gen_feed(): - fg = FeedGenerator() - fg.id(f"{ID}") - fg.title(f"{USERNAME} notes") - fg.author({"name": USERNAME, "email": "t@a4.io"}) - fg.link(href=ID, rel="alternate") - fg.description(f"{USERNAME} notes") - fg.logo(ME.get("icon", {}).get("url")) - fg.language("en") - for item in DB.activities.find( - { - "box": Box.OUTBOX.value, - "type": "Create", - "meta.deleted": False, - "meta.public": True, - }, - limit=10, - ).sort("_id", -1): - fe = fg.add_entry() - fe.id(item["activity"]["object"].get("url")) - fe.link(href=item["activity"]["object"].get("url")) - fe.title(item["activity"]["object"]["content"]) - fe.description(item["activity"]["object"]["content"]) - return fg - - -def json_feed(path: str) -> Dict[str, Any]: - """JSON Feed (https://jsonfeed.org/) document.""" - data = [] - for item in DB.activities.find( - { - "box": Box.OUTBOX.value, - "type": "Create", - "meta.deleted": False, - "meta.public": True, - }, - limit=10, - ).sort("_id", -1): - data.append( - { - "id": item["activity"]["id"], - "url": item["activity"]["object"].get("url"), - "content_html": item["activity"]["object"]["content"], - "content_text": html2text(item["activity"]["object"]["content"]), - "date_published": item["activity"]["object"].get("published"), - } - ) - return { - "version": "https://jsonfeed.org/version/1", - "user_comment": ( - "This is a microblog feed. You can add this to your feed reader using the following URL: " - + ID - + path - ), - "title": USERNAME, - "home_page_url": ID, - "feed_url": ID + path, - "author": { - "name": USERNAME, - "url": ID, - "avatar": ME.get("icon", {}).get("url"), - }, - "items": data, - } - - -def build_inbox_json_feed( - path: str, request_cursor: Optional[str] = None -) -> Dict[str, Any]: - """Build a JSON feed from the inbox activities.""" - data = [] - cursor = None - - q: Dict[str, Any] = { - "type": "Create", - "meta.deleted": False, - "box": Box.INBOX.value, - } - if request_cursor: - q["_id"] = {"$lt": request_cursor} - - for item in DB.activities.find(q, limit=50).sort("_id", -1): - actor = ap.get_backend().fetch_iri(item["activity"]["actor"]) - data.append( - { - "id": item["activity"]["id"], - "url": item["activity"]["object"].get("url"), - "content_html": item["activity"]["object"]["content"], - "content_text": html2text(item["activity"]["object"]["content"]), - "date_published": item["activity"]["object"].get("published"), - "author": { - "name": actor.get("name", actor.get("preferredUsername")), - "url": actor.get("url"), - "avatar": actor.get("icon", {}).get("url"), - }, - } - ) - cursor = str(item["_id"]) - - resp = { - "version": "https://jsonfeed.org/version/1", - "title": f"{USERNAME}'s stream", - "home_page_url": ID, - "feed_url": ID + path, - "items": data, - } - if cursor and len(data) == 50: - resp["next_url"] = ID + path + "?cursor=" + cursor - - return resp - - def embed_collection(total_items, first_page_id): """Helper creating a root OrderedCollection with a link to the first page.""" return { @@ -672,3 +559,41 @@ def build_ordered_collection( # XXX(tsileo): implements prev with prev=? return resp + + +def add_extra_collection(raw_doc: Dict[str, Any]) -> Dict[str, Any]: + if raw_doc["activity"]["type"] != ap.ActivityType.CREATE.value: + return raw_doc + + raw_doc["activity"]["object"]["replies"] = embed_collection( + raw_doc.get("meta", {}).get("count_direct_reply", 0), + f'{raw_doc["remote_id"]}/replies', + ) + + raw_doc["activity"]["object"]["likes"] = embed_collection( + raw_doc.get("meta", {}).get("count_like", 0), f'{raw_doc["remote_id"]}/likes' + ) + + raw_doc["activity"]["object"]["shares"] = embed_collection( + raw_doc.get("meta", {}).get("count_boost", 0), f'{raw_doc["remote_id"]}/shares' + ) + + return raw_doc + + +def remove_context(activity: Dict[str, Any]) -> Dict[str, Any]: + if "@context" in activity: + del activity["@context"] + return activity + + +def activity_from_doc(raw_doc: Dict[str, Any], embed: bool = False) -> Dict[str, Any]: + raw_doc = add_extra_collection(raw_doc) + activity = clean_activity(raw_doc["activity"]) + + # Handle Questions + # TODO(tsileo): what about object embedded by ID/URL? + _add_answers_to_question(raw_doc) + if embed: + return remove_context(activity) + return activity diff --git a/core/feed.py b/core/feed.py new file mode 100644 index 0000000..7082821 --- /dev/null +++ b/core/feed.py @@ -0,0 +1,125 @@ +from typing import Any +from typing import Dict +from typing import Optional + +from feedgen.feed import FeedGenerator +from html2text import html2text +from little_boxes import activitypub as ap + +from config import ID +from config import ME +from config import USERNAME +from core.db import DB +from core.meta import Box + + +def gen_feed(): + fg = FeedGenerator() + fg.id(f"{ID}") + fg.title(f"{USERNAME} notes") + fg.author({"name": USERNAME, "email": "t@a4.io"}) + fg.link(href=ID, rel="alternate") + fg.description(f"{USERNAME} notes") + fg.logo(ME.get("icon", {}).get("url")) + fg.language("en") + for item in DB.activities.find( + { + "box": Box.OUTBOX.value, + "type": "Create", + "meta.deleted": False, + "meta.public": True, + }, + limit=10, + ).sort("_id", -1): + fe = fg.add_entry() + fe.id(item["activity"]["object"].get("url")) + fe.link(href=item["activity"]["object"].get("url")) + fe.title(item["activity"]["object"]["content"]) + fe.description(item["activity"]["object"]["content"]) + return fg + + +def json_feed(path: str) -> Dict[str, Any]: + """JSON Feed (https://jsonfeed.org/) document.""" + data = [] + for item in DB.activities.find( + { + "box": Box.OUTBOX.value, + "type": "Create", + "meta.deleted": False, + "meta.public": True, + }, + limit=10, + ).sort("_id", -1): + data.append( + { + "id": item["activity"]["id"], + "url": item["activity"]["object"].get("url"), + "content_html": item["activity"]["object"]["content"], + "content_text": html2text(item["activity"]["object"]["content"]), + "date_published": item["activity"]["object"].get("published"), + } + ) + return { + "version": "https://jsonfeed.org/version/1", + "user_comment": ( + "This is a microblog feed. You can add this to your feed reader using the following URL: " + + ID + + path + ), + "title": USERNAME, + "home_page_url": ID, + "feed_url": ID + path, + "author": { + "name": USERNAME, + "url": ID, + "avatar": ME.get("icon", {}).get("url"), + }, + "items": data, + } + + +def build_inbox_json_feed( + path: str, request_cursor: Optional[str] = None +) -> Dict[str, Any]: + """Build a JSON feed from the inbox activities.""" + data = [] + cursor = None + + q: Dict[str, Any] = { + "type": "Create", + "meta.deleted": False, + "box": Box.INBOX.value, + } + if request_cursor: + q["_id"] = {"$lt": request_cursor} + + for item in DB.activities.find(q, limit=50).sort("_id", -1): + actor = ap.get_backend().fetch_iri(item["activity"]["actor"]) + data.append( + { + "id": item["activity"]["id"], + "url": item["activity"]["object"].get("url"), + "content_html": item["activity"]["object"]["content"], + "content_text": html2text(item["activity"]["object"]["content"]), + "date_published": item["activity"]["object"].get("published"), + "author": { + "name": actor.get("name", actor.get("preferredUsername")), + "url": actor.get("url"), + "avatar": actor.get("icon", {}).get("url"), + }, + } + ) + cursor = str(item["_id"]) + + resp = { + "version": "https://jsonfeed.org/version/1", + "title": f"{USERNAME}'s stream", + "home_page_url": ID, + "feed_url": ID + path, + "items": data, + } + if cursor and len(data) == 50: + resp["next_url"] = ID + path + "?cursor=" + cursor + + return resp