From 43e113e420a19becdc8a0cb9a62e98ae85d05f07 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 18 May 2018 20:41:41 +0200 Subject: [PATCH 0001/1425] Initial import --- .dockerignore | 1 + .gitignore | 8 + .travis.yml | 9 + Dockerfile | 6 + LICENSE | 661 +++++++++++++++ Makefile | 5 + README.md | 74 ++ activitypub.py | 978 +++++++++++++++++++++++ app.py | 1014 ++++++++++++++++++++++++ config.py | 83 ++ config/me.sample.yml | 6 + data/.gitignore | 2 + docker-compose-dev.yml | 21 + docker-compose.yml | 30 + requirements.txt | 21 + sass/base_theme.scss | 207 +++++ sass/theme.scss | 1 + setup.cfg | 2 + static/base.css | 160 ++++ static/css/.gitignore | 2 + static/media/.gitignore | 2 + static/nopic.png | Bin 0 -> 1791 bytes tasks.py | 56 ++ templates/about.html | 22 + templates/admin.html | 30 + templates/authorize_remote_follow.html | 15 + templates/followers.html | 16 + templates/following.html | 16 + templates/header.html | 29 + templates/index.html | 45 ++ templates/indieauth_flow.html | 42 + templates/layout.html | 29 + templates/login.html | 36 + templates/new.html | 17 + templates/note.html | 23 + templates/remote_follow.html | 16 + templates/stream.html | 39 + templates/tags.html | 30 + templates/u2f.html | 29 + templates/utils.html | 91 +++ utils/__init__.py | 0 utils/actor_service.py | 72 ++ utils/content_helper.py | 57 ++ utils/httpsig.py | 87 ++ utils/key.py | 39 + utils/linked_data_sig.py | 53 ++ utils/object_service.py | 60 ++ utils/opengraph.py | 46 ++ utils/urlutils.py | 27 + utils/webfinger.py | 63 ++ 50 files changed, 4378 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 activitypub.py create mode 100644 app.py create mode 100644 config.py create mode 100644 config/me.sample.yml create mode 100644 data/.gitignore create mode 100644 docker-compose-dev.yml create mode 100644 docker-compose.yml create mode 100644 requirements.txt create mode 100644 sass/base_theme.scss create mode 100644 sass/theme.scss create mode 100644 setup.cfg create mode 100644 static/base.css create mode 100644 static/css/.gitignore create mode 100644 static/media/.gitignore create mode 100644 static/nopic.png create mode 100644 tasks.py create mode 100644 templates/about.html create mode 100644 templates/admin.html create mode 100644 templates/authorize_remote_follow.html create mode 100644 templates/followers.html create mode 100644 templates/following.html create mode 100644 templates/header.html create mode 100644 templates/index.html create mode 100644 templates/indieauth_flow.html create mode 100644 templates/layout.html create mode 100644 templates/login.html create mode 100644 templates/new.html create mode 100644 templates/note.html create mode 100644 templates/remote_follow.html create mode 100644 templates/stream.html create mode 100644 templates/tags.html create mode 100644 templates/u2f.html create mode 100644 templates/utils.html create mode 100644 utils/__init__.py create mode 100644 utils/actor_service.py create mode 100644 utils/content_helper.py create mode 100644 utils/httpsig.py create mode 100644 utils/key.py create mode 100644 utils/linked_data_sig.py create mode 100644 utils/object_service.py create mode 100644 utils/opengraph.py create mode 100644 utils/urlutils.py create mode 100644 utils/webfinger.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8fce603 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +data/ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e43f899 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +*.sw[op] +key_*.pem +data/* +config/* +static/media/* + +.mypy_cache/ +__pycache__/ diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..e3b0785 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,9 @@ +language: python +python: + - "3.6" +install: + - pip install pytest mypy flake8 +script: + - flake8 + - mypy --ignore-missing-imports + - pytest -v diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3fed815 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.6 +ADD . /app +WORKDIR /app +RUN pip install -r requirements.txt +ENV FLASK_APP=app.py +CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5005", "app:app"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..be3f7b2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..bc473fc --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ +css: + python -c "import sass; sass.compile(dirname=('sass', 'static/css'), output_style='compressed')" + +password: + python -c "import bcrypt; from getpass import getpass; print(bcrypt.hashpw(getpass().encode('utf-8'), bcrypt.gensalt()).decode('utf-8'))" diff --git a/README.md b/README.md new file mode 100644 index 0000000..071be44 --- /dev/null +++ b/README.md @@ -0,0 +1,74 @@ +# microblog.pub + +

+ microblog.pub +

+

+Version +Build Status +License +

+ +

A self-hosted, single-user, ActivityPub powered microblog.

+ +## Features + + - Implements a basic [ActivityPub](https://activitypub.rocks/) server (with federation) + - Compatible with [Mastodon](https://github.com/tootsuite/mastodon) and others (Pleroma, Hubzilla...) + - Also implements a remote follow compatible with Mastodon instances + - Expose your outbox as a basic microblog + - [IndieAuth](https://indieauth.spec.indieweb.org/) endpoints (authorization and token endpoint) + - U2F support + - You can use your ActivityPub identity to login to other websites/app + - Admin UI with notifications and the stream of people you follow + - Attach files to your notes + - Privacy-aware upload that strip EXIF meta data before storing the file + - No JavaScript, that's it, even the admin UI is pure HTML/CSS + - Easy to customize (the theme is written Sass) + - Microformats aware (exports `h-feed`, `h-entry`, `h-cards`, ...) + - Exports RSS/Atom feeds + - Comes with a tiny HTTP API to help posting new content and performing basic actions + - Deployable with Docker + +## Running your instance + +### Installation + +```shell +$ git clone +$ make css +``` + +### Configuration + +```shell +$ make password +``` + +### Deployment + +```shell +$ docker-compose up -d +``` + +You should use a reverse proxy... + +## Development + +The most convenient way to hack on microblog.pub is to run the server locally, and run + + +```shell +# One-time setup +$ pip install -r requirements.txt +# Start the Celery worker, RabbitMQ and MongoDB +$ docker-compose -f docker-compose-dev.yml up -d +# Run the server locally +$ FLASK_APP=app.py flask run -p 5005 --with-threads +``` + +## Contributions + +PRs are welcome, please open an issue to start a discussion before your start any work. diff --git a/activitypub.py b/activitypub.py new file mode 100644 index 0000000..2a4a937 --- /dev/null +++ b/activitypub.py @@ -0,0 +1,978 @@ +import typing +import re +import json +import binascii +import os +from datetime import datetime +from enum import Enum + +import requests +from bleach.linkifier import Linker +from bson.objectid import ObjectId +from html2text import html2text +from feedgen.feed import FeedGenerator +from markdown import markdown + +from utils.linked_data_sig import generate_signature +from utils.actor_service import NotAnActorError +from utils.webfinger import get_actor_url +from utils.content_helper import parse_markdown +from config import USERNAME, BASE_URL, ID +from config import CTX_AS, CTX_SECURITY, AS_PUBLIC +from config import KEY, DB, ME, ACTOR_SERVICE +from config import OBJECT_SERVICE +from config import PUBLIC_INSTANCES +import tasks + +from typing import List, Optional, Tuple, Dict, Any, Union, Type +from typing import TypeVar + +A = TypeVar('A', bound='BaseActivity') +ObjectType = Dict[str, Any] +ObjectOrIDType = Union[str, ObjectType] + + +# Pleroma sample +# {'@context': ['https://www.w3.org/ns/activitystreams', 'https://w3id.org/security/v1', {'Emoji': 'toot:Emoji', 'Hashtag': 'as:Hashtag', 'atomUri': 'ostatus:atomUri', 'conversation': 'ostatus:conversation', 'inReplyToAtomUri': 'ostatus:inReplyToAtomUri', 'manuallyApprovesFollowers': 'as:manuallyApprovesFollowers', 'ostatus': 'http://ostatus.org#', 'sensitive': 'as:sensitive', 'toot': 'http://joinmastodon.org/ns#'}], 'actor': 'https://soc.freedombone.net/users/bob', 'attachment': [{'mediaType': 'image/jpeg', 'name': 'stallmanlemote.jpg', 'type': 'Document', 'url': 'https://soc.freedombone.net/media/e1a3ca6f-df73-4f2d-a931-c389a221b008/stallmanlemote.jpg'}], 'attributedTo': 'https://soc.freedombone.net/users/bob', 'cc': ['https://cybre.space/users/vantablack', 'https://soc.freedombone.net/users/bob/followers'], 'content': '@vantablack
stallmanlemote.jpg', 'context': 'tag:cybre.space,2018-04-05:objectId=5756519:objectType=Conversation', 'conversation': 'tag:cybre.space,2018-04-05:objectId=5756519:objectType=Conversation', 'emoji': {}, 'id': 'https://soc.freedombone.net/objects/3f0faeca-4d37-4acf-b990-6a50146d23cc', 'inReplyTo': 'https://cybre.space/users/vantablack/statuses/99808953472969467', 'inReplyToStatusId': 300713, 'like_count': 1, 'likes': ['https://cybre.space/users/vantablack'], 'published': '2018-04-05T21:30:52.658817Z', 'sensitive': False, 'summary': None, 'tag': [{'href': 'https://cybre.space/users/vantablack', 'name': '@vantablack@cybre.space', 'type': 'Mention'}], 'to': ['https://www.w3.org/ns/activitystreams#Public'], 'type': 'Note'} + + +class ActivityTypes(Enum): + ANNOUNCE = 'Announce' + BLOCK = 'Block' + LIKE = 'Like' + CREATE = 'Create' + UPDATE = 'Update' + PERSON = 'Person' + ORDERED_COLLECTION = 'OrderedCollection' + ORDERED_COLLECTION_PAGE = 'OrderedCollectionPage' + COLLECTION_PAGE = 'CollectionPage' + COLLECTION = 'Collection' + NOTE = 'Note' + ACCEPT = 'Accept' + REJECT = 'Reject' + FOLLOW = 'Follow' + DELETE = 'Delete' + UNDO = 'Undo' + IMAGE = 'Image' + TOMBSTONE = 'Tombstone' + + +def random_object_id() -> str: + return binascii.hexlify(os.urandom(8)).decode('utf-8') + + +def _remove_id(doc: ObjectType) -> ObjectType: + doc = doc.copy() + if '_id' in doc: + del(doc['_id']) + return doc + + +def _to_list(data: Union[List[Any], Any]) -> List[Any]: + if isinstance(data, list): + return data + return [data] + + +def clean_activity(activity: ObjectType) -> Dict[str, Any]: + # Remove the hidden bco and bcc field + for field in ['bto', 'bcc']: + if field in activity: + del(activity[field]) + if activity['type'] == 'Create' and field in activity['object']: + del(activity['object'][field]) + return activity + + +def _get_actor_id(actor: ObjectOrIDType) -> str: + if isinstance(actor, dict): + return actor['id'] + return actor + + +class BaseActivity(object): + ACTIVITY_TYPE = None # type: Optional[ActivityTypes] + NO_CONTEXT = False + ALLOWED_OBJECT_TYPES = None # type: List[ActivityTypes] + + def __init__(self, **kwargs) -> None: + if not self.ACTIVITY_TYPE: + raise ValueError('Missing ACTIVITY_TYPE') + + if kwargs.get('type') is not None and kwargs.pop('type') != self.ACTIVITY_TYPE.value: + raise ValueError('Expect the type to be {}'.format(self.ACTIVITY_TYPE)) + + self._data = {'type': self.ACTIVITY_TYPE.value} # type: Dict[str, Any] + + if 'id' in kwargs: + self._data['id'] = kwargs.pop('id') + + if self.ACTIVITY_TYPE != ActivityTypes.PERSON: + actor = kwargs.get('actor') + if actor: + kwargs.pop('actor') + actor = self._validate_person(actor) + self._data['actor'] = actor + else: + if not self.NO_CONTEXT: + actor = ID + self._data['actor'] = actor + + if 'object' in kwargs: + obj = kwargs.pop('object') + if isinstance(obj, str): + self._data['object'] = obj + else: + if not self.ALLOWED_OBJECT_TYPES: + raise ValueError('unexpected object') + if 'type' not in obj or (self.ACTIVITY_TYPE != ActivityTypes.CREATE and 'id' not in obj): + raise ValueError('invalid object') + if ActivityTypes(obj['type']) not in self.ALLOWED_OBJECT_TYPES: + print(self, kwargs) + raise ValueError(f'unexpected object type {obj["type"]} (allowed={self.ALLOWED_OBJECT_TYPES})') + self._data['object'] = obj + + if '@context' not in kwargs: + if not self.NO_CONTEXT: + self._data['@context'] = CTX_AS + else: + self._data['@context'] = kwargs.pop('@context') + + # @context check + if not self.NO_CONTEXT: + if not isinstance(self._data['@context'], list): + self._data['@context'] = [self._data['@context']] + if not CTX_SECURITY in self._data['@context']: + self._data['@context'].append(CTX_SECURITY) + if isinstance(self._data['@context'][-1], dict): + self._data['@context'][-1]['Hashtag'] = 'as:Hashtag' + self._data['@context'][-1]['sensitive'] = 'as:sensitive' + else: + self._data['@context'].append({'Hashtag': 'as:Hashtag', 'sensitive': 'as:sensitive'}) + + allowed_keys = None + try: + allowed_keys = self._init(**kwargs) + except NotImplementedError: + pass + + if allowed_keys: + # Allows an extra to (like for Accept and Follow) + kwargs.pop('to', None) + if len(set(kwargs.keys()) - set(allowed_keys)) > 0: + raise ValueError('extra data left: {}'.format(kwargs)) + else: + self._data.update(**kwargs) + + def _init(self, **kwargs) -> Optional[List[str]]: + raise NotImplementedError + + def _verify(self) -> None: + raise NotImplementedError + + def verify(self) -> None: + try: + self._verify() + except NotImplementedError: + pass + + def __repr__(self) -> str: + return '{}({!r})'.format(self.__class__.__qualname__, self._data.get('id')) + + def __str__(self) -> str: + return str(self._data['id']) + + def __getattr__(self, name: str) -> Any: + if self._data.get(name): + return self._data.get(name) + + @property + def type_enum(self) -> ActivityTypes: + return ActivityTypes(self.type) + + def set_id(self, uri: str, obj_id: str) -> None: + self._data['id'] = uri + + def _actor_id(self, obj: ObjectOrIDType) -> str: + if isinstance(obj, dict) and obj['type'] == ActivityTypes.PERSON.value: + obj_id = obj.get('id') + if not obj_id: + raise ValueError('missing object id') + return obj_id + else: + return str(obj) + + def _validate_person(self, obj: ObjectOrIDType) -> str: + obj_id = self._actor_id(obj) + try: + actor = ACTOR_SERVICE.get(obj_id) + except Exception: + return obj_id # FIXME(tsileo): handle this + if not actor: + raise ValueError('Invalid actor') + return actor['id'] + + def get_object(self) -> 'BaseActivity': + if self.__obj: + return self.__obj + if isinstance(self._data['object'], dict): + p = parse_activity(self._data['object']) + else: + if self.ACTIVITY_TYPE == ActivityTypes.FOLLOW: + p = Person(**ACTOR_SERVICE.get(self._data['object'])) + else: + obj = OBJECT_SERVICE.get(self._data['object']) + if ActivityTypes(obj.get('type')) not in self.ALLOWED_OBJECT_TYPES: + raise ValueError('invalid object type') + + p = parse_activity(obj) + + self.__obj = p # type: BaseActivity + return p + + def _to_dict(self, data: ObjectType) -> ObjectType: + return data + + def to_dict(self, embed: bool = False) -> ObjectType: + data = dict(self._data) + if embed: + for k in ['@context', 'signature']: + if k in data: + del(data[k]) + return self._to_dict(data) + + def get_actor(self) -> 'BaseActivity': + actor = self._data.get('actor') + if not actor: + if self.type_enum == ActivityTypes.NOTE: + actor = str(self._data.get('attributedTo')) + else: + raise ValueError('failed to fetch actor') + + actor_id = self._actor_id(actor) + return Person(**ACTOR_SERVICE.get(actor_id)) + + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: + raise NotImplementedError + + def _undo_outbox(self) -> None: + raise NotImplementedError + + def _process_from_inbox(self) -> None: + raise NotImplementedError + + def _undo_inbox(self) -> None: + raise NotImplementedError + + def process_from_inbox(self) -> None: + self.verify() + actor = self.get_actor() + + if DB.outbox.find_one({'type': ActivityTypes.BLOCK.value, + 'activity.object': actor.id, + 'meta.undo': False}): + print('actor is blocked, drop activity') + return + + if DB.inbox.find_one({'remote_id': self.id}): + # The activity is already in the inbox + print('received duplicate activity') + return + + activity = self.to_dict() + DB.inbox.insert_one({ + 'activity': activity, + 'type': self.type, + 'remote_id': self.id, + 'meta': {'undo': False, 'deleted': False}, + }) + + try: + self._process_from_inbox() + except NotImplementedError: + pass + + def post_to_outbox(self) -> None: + obj_id = random_object_id() + self.set_id(f'{ID}/outbox/{obj_id}', obj_id) + self.verify() + activity = self.to_dict() + DB.outbox.insert_one({ + 'id': obj_id, + 'activity': activity, + 'type': self.type, + 'remote_id': self.id, + 'meta': {'undo': False, 'deleted': False}, + }) + + recipients = self.recipients() + activity = clean_activity(activity) + + try: + self._post_to_outbox(obj_id, activity, recipients) + except NotImplementedError: + pass + + #return + generate_signature(activity, KEY.privkey) + payload = json.dumps(activity) + print('will post') + for recp in recipients: + self._post_to_inbox(payload, recp) + print('done') + + def _post_to_inbox(self, payload: str, to: str): + tasks.post_to_inbox.delay(payload, to) + + def _recipients(self) -> List[str]: + return [] + + def recipients(self) -> List[str]: + recipients = self._recipients() + + out = [] # type: List[str] + for recipient in recipients: + if recipient in PUBLIC_INSTANCES: + if recipient not in out: + out.append(str(recipient)) + continue + if recipient in [ME, AS_PUBLIC, None]: + continue + if isinstance(recipient, Person): + if recipient.id == ME: + continue + actor = recipient + else: + try: + actor = Person(**ACTOR_SERVICE.get(recipient)) + except NotAnActorError as error: + # Is the activity a `Collection`/`OrderedCollection`? + if error.activity and error.activity['type'] in [ActivityTypes.COLLECTION.value, + ActivityTypes.ORDERED_COLLECTION.value]: + for item in parse_collection(error.activity): + if item in [ME, AS_PUBLIC]: + continue + try: + col_actor = Person(**ACTOR_SERVICE.get(item)) + except NotAnActorError: + pass + + if col_actor.endpoints: + shared_inbox = col_actor.endpoints.get('sharedInbox') + if shared_inbox not in out: + out.append(shared_inbox) + continue + if col_actor.inbox and col_actor.inbox not in out: + out.append(col_actor.inbox) + + continue + + if actor.endpoints: + shared_inbox = actor.endpoints.get('sharedInbox') + if shared_inbox not in out: + out.append(shared_inbox) + continue + + if actor.inbox and actor.inbox not in out: + out.append(actor.inbox) + + return out + + def build_undo(self) -> 'BaseActivity': + raise NotImplementedError + + +class Person(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.PERSON + + def _init(self, **kwargs): + #if 'icon' in kwargs: + # self._data['icon'] = Image(**kwargs.pop('icon')) + pass + + def _verify(self) -> None: + ACTOR_SERVICE.get(self._data['id']) + + def _to_dict(self, data): + #if 'icon' in data: + # data['icon'] = data['icon'].to_dict() + # + return data + + +class Block(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.BLOCK + + +class Collection(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.COLLECTION + + +class Image(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.IMAGE + NO_CONTEXT = True + + def _init(self, **kwargs): + self._data.update( + url=kwargs.pop('url'), + ) + + def __repr__(self): + return 'Image({!r})'.format(self._data.get('url')) + + +class Follow(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.FOLLOW + ALLOWED_OBJECT_TYPES = [ActivityTypes.PERSON] + + def _build_reply(self, reply_type: ActivityTypes) -> BaseActivity: + if reply_type == ActivityTypes.ACCEPT: + return Accept( + object=self.to_dict(embed=True), + ) + + raise ValueError(f'type {reply_type} is invalid for building a reply') + + def _recipients(self) -> List[str]: + return [self.get_object().id] + + def _process_from_inbox(self) -> None: + accept = self.build_accept() + accept.post_to_outbox() + + remote_actor = self.get_actor().id + + if DB.followers.find({'remote_actor': remote_actor}).count() == 0: + DB.followers.insert_one({'remote_actor': remote_actor}) + + def _undo_inbox(self) -> None: + DB.followers.delete_one({'remote_actor': self.get_actor().id}) + + def build_accept(self) -> BaseActivity: + return self._build_reply(ActivityTypes.ACCEPT) + + def build_undo(self) -> BaseActivity: + return Undo(object=self.to_dict(embed=True)) + + +class Accept(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.ACCEPT + ALLOWED_OBJECT_TYPES = [ActivityTypes.FOLLOW] + + def _recipients(self) -> List[str]: + return [self.get_object().get_actor().id] + + def _process_from_inbox(self) -> None: + remote_actor = self.get_actor().id + if DB.following.find({'remote_actor': remote_actor}).count() == 0: + DB.following.insert_one({'remote_actor': remote_actor}) + + +class Undo(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.UNDO + ALLOWED_OBJECT_TYPES = [ActivityTypes.FOLLOW, ActivityTypes.LIKE, ActivityTypes.ANNOUNCE] + + def _recipients(self) -> List[str]: + obj = self.get_object() + if obj.type_enum == ActivityTypes.FOLLOW: + return [obj.get_object().id] + else: + return [obj.get_object().get_actor().id] + # TODO(tsileo): handle like and announce + raise Exception('TODO') + + def _process_from_inbox(self) -> None: + obj = self.get_object() + DB.inbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) + + try: + obj._undo_inbox() + except NotImplementedError: + pass + + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: + obj = self.get_object() + DB.outbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) + + try: + obj._undo_outbox() + except NotImplementedError: + pass + +class Like(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.LIKE + ALLOWED_OBJECT_TYPES = [ActivityTypes.NOTE] + + def _recipients(self) -> List[str]: + return [self.get_object().get_actor().id] + + def _process_from_inbox(self): + obj = self.get_object() + # Update the meta counter if the object is published by the server + DB.outbox.update_one({'activity.object.id': obj.id}, {'$inc': {'meta.count_like': 1}}) + + def _undo_inbox(self) -> None: + obj = self.get_object() + # Update the meta counter if the object is published by the server + DB.outbox.update_one({'activity.object.id': obj.id}, {'$inc': {'meta.count_like': -1}}) + + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]): + obj = self.get_object() + # Unlikely, but an actor can like it's own post + DB.outbox.update_one({'activity.object.id': obj.id}, {'$inc': {'meta.count_like': 1}}) + + DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.liked': obj_id}}) + + def _undo_outbox(self) -> None: + obj = self.get_object() + # Unlikely, but an actor can like it's own post + DB.outbox.update_one({'activity.object.id': obj.id}, {'$inc': {'meta.count_like': -1}}) + + DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.liked': False}}) + + def build_undo(self) -> BaseActivity: + return Undo(object=self.to_dict(embed=True)) + + +class Announce(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.ANNOUNCE + ALLOWED_OBJECT_TYPES = [ActivityTypes.NOTE] + + def _recipients(self) -> List[str]: + recipients = [] + + for field in ['to', 'cc']: + if field in self._data: + recipients.extend(_to_list(self._data[field])) + + return recipients + + def _process_from_inbox(self) -> None: + if isinstance(self._data['object'], str) and not self._data['object'].startswith('http'): + # TODO(tsileo): actually drop it without storing it and better logging, also move the check somewhere else + print(f'received an Annouce referencing an OStatus notice ({self._data["object"]}), dropping the message') + return + # Save/cache the object, and make it part of the stream so we can fetch it + if isinstance(self._data['object'], str): + raw_obj = OBJECT_SERVICE.get(self._data['object'], reload_cache=True, part_of_stream=True, announce_published=self._data['published']) + obj = parse_activity(raw_obj) + else: + obj = self.get_object() + DB.outbox.update_one({'activity.object.id': obj.id}, {'$inc': {'meta.count_boost': 1}}) + + def _undo_inbox(self) -> None: + obj = self.get_object() + DB.inbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) + DB.outbox.update_one({'activity.object.id': obj.id}, {'$inc': {'meta.count_boost': -1}}) + + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: + if isinstance(self._data['object'], str): + # Put the object in the cache + OBJECT_SERVICE.get(self._data['object'], reload_cache=True, part_of_stream=True, announce_published=self._data['published']) + + obj = self.get_object() + DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.boosted': obj_id}}) + + def _undo_outbox(self) -> None: + obj = self.get_object() + DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.boosted': False}}) + + def build_undo(self) -> BaseActivity: + return Undo(object=self.to_dict(embed=True)) + + +class Delete(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.DELETE + ALLOWED_OBJECT_TYPES = [ActivityTypes.NOTE, ActivityTypes.TOMBSTONE] + + def _recipients(self) -> List[str]: + return self.get_object().recipients() + + def _process_from_inbox(self): + DB.inbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) + # TODO(tsileo): also delete copies stored in parents' `meta.replies` + + def _post_to_outbox(self, obj_id, activity, recipients): + DB.outbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) + + +class Update(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.UPDATE + ALLOWED_OBJECT_TYPES = [ActivityTypes.NOTE, ActivityTypes.PERSON] + + # TODO(tsileo): ensure the actor updating is the same as the orinial activity + # (ensuring that the Update and its object are of same origin) + + def _process_from_inbox(self): + obj = self.get_object() + if obj.type_enum == ActivityTypes.NOTE: + DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'activity.object': obj.to_dict()}}) + return + + # If the object is a Person, it means the profile was updated, we just refresh our local cache + ACTOR_SERVICE.get(obj.id, reload_cache=True) + + def _post_to_outbox(self, obj_id, activity, recipients): + obj = self.get_object() + + update_prefix = 'activity.object.' + update = {'$set': dict(), '$unset': dict()} + update['$set'][f'{update_prefix}updated'] = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' + for k, v in obj._data.items(): + if k in ['id', 'type']: + continue + if v is None: + update['$unset'][f'{update_prefix}{k}'] = '' + else: + update['$set'][f'{update_prefix}{k}'] = v + + if len(update['$unset']) == 0: + del(update['$unset']) + + DB.outbox.update_one({'remote_id': obj.id.replace('/activity', '')}, update) + # FIXME(tsileo): should send an Update (but not a partial one, to all the note's recipients + # (create a new Update with the result of the update, and send it without saving it?) + + +class Create(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.CREATE + ALLOWED_OBJECT_TYPES = [ActivityTypes.NOTE] + + def _set_id(self, uri, obj_id): + self._data['object']['id'] = uri + '/activity' + self._data['object']['url'] = ID + '/' + self.get_object().type.lower() + '/' + obj_id + + def _init(self, **kwargs): + obj = self.get_object() + if not obj.attributedTo: + self._data['object']['attributedTo'] = self.get_actor().id + if not obj.published: + if self.published: + self._data['object']['published'] = self.published + else: + now = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' + self._data['published'] = now + self._data['object']['published'] = now + + def _recipients(self) -> List[str]: + # TODO(tsileo): audience support? + recipients = [] + for field in ['to', 'cc', 'bto', 'bcc']: + if field in self._data: + recipients.extend(_to_list(self._data[field])) + + recipients.extend(self.get_object()._recipients()) + + return recipients + + def _process_from_inbox(self): + obj = self.get_object() + + tasks.fetch_og.delay('INBOX', self.id) + + in_reply_to = obj.inReplyTo + if in_reply_to: + parent = DB.inbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) + if not parent: + DB.outbox.update_one( + {'activity.object.id': in_reply_to}, + {'$inc': {'meta.count_reply': 1}}, + ) + return + + # If the note is a "reply of a reply" update the parent message + # TODO(tsileo): review this code + while parent: + DB.inbox.update_one({'_id': parent['_id']}, {'$push': {'meta.replies': self.to_dict()}}) + in_reply_to = parent.get('activity', {}).get('object', {}).get('inReplyTo') + if in_reply_to: + parent = DB.inbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) + if parent is None: + # The reply is a note from the outbox + data = DB.outbox.update_one( + {'activity.object.id': in_reply_to}, + {'$inc': {'meta.count_reply': 1}}, + ) + else: + parent = None + + +class Tombstone(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.TOMBSTONE + + +class Note(BaseActivity): + ACTIVITY_TYPE = ActivityTypes.NOTE + + def _init(self, **kwargs): + print(self._data) + # Remove the `actor` field as `attributedTo` is used for `Note` instead + if 'actor' in self._data: + del(self._data['actor']) + # FIXME(tsileo): use kwarg + # TODO(tsileo): support mention tag + # TODO(tisleo): implement the tag endpoint + if 'sensitive' not in kwargs: + self._data['sensitive'] = False + + # FIXME(tsileo): add the tag in CC + # for t in kwargs.get('tag', []): + # if t['type'] == 'Mention': + # cc -> c['href'] + + def _recipients(self) -> List[str]: + # TODO(tsileo): audience support? + recipients = [] # type: List[str] + + # If the note is public, we publish it to the defined "public instances" + if AS_PUBLIC in self._data.get('to', []): + recipients.extend(PUBLIC_INSTANCES) + print('publishing to public instances') + print(recipients) + + for field in ['to', 'cc', 'bto', 'bcc']: + if field in self._data: + recipients.extend(_to_list(self._data[field])) + + return recipients + + def build_create(self) -> BaseActivity: + """Wraps an activity in a Create activity.""" + create_payload = { + 'object': self.to_dict(embed=True), + 'actor': self.attributedTo or ME, + } + for field in ['published', 'to', 'bto', 'cc', 'bcc', 'audience']: + if field in self._data: + create_payload[field] = self._data[field] + + return Create(**create_payload) + + def build_like(self) -> BaseActivity: + return Like(object=self.id) + + def build_announce(self) -> BaseActivity: + return Announce( + object=self.id, + to=[AS_PUBLIC], + cc=[ID+'/followers', self.attributedTo], + published=datetime.utcnow().replace(microsecond=0).isoformat() + 'Z', + ) + +_ACTIVITY_TYPE_TO_CLS = { + ActivityTypes.IMAGE: Image, + ActivityTypes.PERSON: Person, + ActivityTypes.FOLLOW: Follow, + ActivityTypes.ACCEPT: Accept, + ActivityTypes.UNDO: Undo, + ActivityTypes.LIKE: Like, + ActivityTypes.ANNOUNCE: Announce, + ActivityTypes.UPDATE: Update, + ActivityTypes.DELETE: Delete, + ActivityTypes.CREATE: Create, + ActivityTypes.NOTE: Note, + ActivityTypes.BLOCK: Block, + ActivityTypes.COLLECTION: Collection, + ActivityTypes.TOMBSTONE: Tombstone, +} + +def parse_activity(payload: ObjectType) -> BaseActivity: + t = ActivityTypes(payload['type']) + if t not in _ACTIVITY_TYPE_TO_CLS: + raise ValueError('unsupported activity type') + + return _ACTIVITY_TYPE_TO_CLS[t](**payload) + + +def gen_feed(): + fg = FeedGenerator() + fg.id(f'{ID}') + fg.title(f'{USERNAME} notes') + fg.author( {'name': USERNAME,'email':'t@a4.io'} ) + fg.link(href=ID, rel='alternate') + fg.description(f'{USERNAME} notes') + fg.logo(ME.get('icon', {}).get('url')) + #fg.link( href='http://larskiesow.de/test.atom', rel='self' ) + fg.language('en') + for item in DB.outbox.find({'type': 'Create'}, limit=50): + fe = fg.add_entry() + fe.id(item['activity']['object'].get('url')) + fe.link(href=item['activity']['object'].get('url')) + fe.title(item['activity']['object']['content']) + fe.description(item['activity']['object']['content']) + return fg + + +def json_feed(path: str) -> Dict[str, Any]: + """JSON Feed (https://jsonfeed.org/) document.""" + data = [] + for item in DB.outbox.find({'type': 'Create'}, limit=50): + data.append({ + "id": item["id"], + "url": item['activity']['object'].get('url'), + "content_html": item['activity']['object']['content'], + "content_text": html2text(item['activity']['object']['content']), + "date_published": item['activity']['object'].get('published'), + }) + return { + "version": "https://jsonfeed.org/version/1", + "user_comment": "This is a microblog feed. You can add this to your feed reader using the following URL: " + ID + path, + "title": USERNAME, + "home_page_url": ID, + "feed_url": ID + path, + "author": { + "name": USERNAME, + "url": ID, + "avatar": ME.get('icon', {}).get('url'), + }, + "items": data, + } + + +def build_inbox_json_feed(path: str, request_cursor: Optional[str] = None) -> Dict[str, Any]: + data = [] + cursor = None + + q = {'type': 'Create'} # type: Dict[str, Any] + if request_cursor: + q['_id'] = {'$lt': request_cursor} + + for item in DB.inbox.find(q, limit=50).sort('_id', -1): + actor = ACTOR_SERVICE.get(item['activity']['actor']) + data.append({ + "id": item["activity"]["id"], + "url": item['activity']['object'].get('url'), + "content_html": item['activity']['object']['content'], + "content_text": html2text(item['activity']['object']['content']), + "date_published": item['activity']['object'].get('published'), + "author": { + "name": actor.get('name', actor.get('preferredUsername')), + "url": actor.get('url'), + 'avatar': actor.get('icon', {}).get('url'), + }, + }) + cursor = str(item['_id']) + + resp = { + "version": "https://jsonfeed.org/version/1", + "title": f'{USERNAME}\'s stream', + "home_page_url": ID, + "feed_url": ID + path, + "items": data, + } + if cursor and len(data) == 50: + resp['next_url'] = ID + path + '?cursor=' + cursor + + return resp + + +def parse_collection(payload: Optional[Dict[str, Any]] = None, url: Optional[str] = None) -> List[str]: + """Resolve/fetch a `Collection`/`OrderedCollection`.""" + # Resolve internal collections via MongoDB directly + if url == ID + '/followers': + return [doc['remote_actor'] for doc in DB.followers.find()] + elif url == ID + '/following': + return [doc['remote_actor'] for doc in DB.following.find()] + + # Go through all the pages + out = [] # type: List[str] + if url: + resp = requests.get(url, headers={'Accept': 'application/activity+json'}) + resp.raise_for_status() + payload = resp.json() + + if not payload: + raise ValueError('must at least prove a payload or an URL') + + if payload['type'] in ['Collection', 'OrderedCollection']: + if 'orderedItems' in payload: + return payload['orderedItems'] + if 'items' in payload: + return payload['items'] + if 'first' in payload: + if 'orderedItems' in payload['first']: + out.extend(payload['first']['orderedItems']) + if 'items' in payload['first']: + out.extend(payload['first']['items']) + n = payload['first'].get('next') + if n: + out.extend(parse_collection(url=n)) + return out + + while payload: + if payload['type'] in ['CollectionPage', 'OrderedCollectionPage']: + if 'orderedItems' in payload: + out.extend(payload['orderedItems']) + if 'items' in payload: + out.extend(payload['items']) + n = payload.get('next') + if n is None: + break + resp = requests.get(n, headers={'Accept': 'application/activity+json'}) + resp.raise_for_status() + payload = resp.json() + else: + raise Exception('unexpected activity type {}'.format(payload['type'])) + + return out + + +def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, col_name=None): + col_name = col_name or col.name + if q is None: + q = {} + + if cursor: + q['_id'] = {'$lt': ObjectId(cursor)} + data = list(col.find(q, limit=limit).sort('_id', -1)) + + if not data: + return { + 'id': BASE_URL + '/' + col_name, + 'totalItems': 0, + 'type': 'OrderedCollection', + 'orederedItems': [], + } + + start_cursor = str(data[0]['_id']) + next_page_cursor = str(data[-1]['_id']) + total_items = col.find(q).count() + + data = [_remove_id(doc) for doc in data] + if map_func: + data = [map_func(doc) for doc in data] + + # No cursor, this is the first page and we return an OrderedCollection + if not cursor: + resp = { + '@context': CTX_AS, + 'first': { + 'id': BASE_URL + '/' + col_name + '?cursor=' + start_cursor, + 'orderedItems': data, + 'partOf': BASE_URL + '/' + col_name, + 'totalItems': total_items, + 'type': 'OrderedCollectionPage' + }, + 'id': BASE_URL + '/' + col_name, + 'totalItems': total_items, + 'type': 'OrderedCollection' + } + + if len(data) == limit: + resp['first']['next'] = BASE_URL + '/' + col_name + '?cursor=' + next_page_cursor + + return resp + + # If there's a cursor, then we return an OrderedCollectionPage + resp = { + '@context': CTX_AS, + 'type': 'OrderedCollectionPage', + 'id': BASE_URL + '/' + col_name + '?cursor=' + start_cursor, + 'totalItems': total_items, + 'partOf': BASE_URL + '/' + col_name, + 'orderedItems': data, + } + if len(data) == limit: + resp['next'] = BASE_URL + '/' + col_name + '?cursor=' + next_page_cursor + + return resp diff --git a/app.py b/app.py new file mode 100644 index 0000000..034f1ca --- /dev/null +++ b/app.py @@ -0,0 +1,1014 @@ +import binascii +import hashlib +import json +import urllib +import os +import mimetypes +import logging +from functools import wraps +from datetime import datetime + +import timeago +import bleach +import mf2py +import pymongo +import piexif +from bson.objectid import ObjectId +from flask import Flask +from flask import abort +from flask import request +from flask import redirect +from flask import Response +from flask import render_template +from flask import session +from flask import url_for +from html2text import html2text +from itsdangerous import JSONWebSignatureSerializer +from itsdangerous import BadSignature +from passlib.hash import bcrypt +from u2flib_server import u2f +from urllib.parse import urlparse, urlencode +from werkzeug.utils import secure_filename + +import activitypub +import config +from activitypub import ActivityTypes +from activitypub import clean_activity +from activitypub import parse_markdown +from config import KEY +from config import DB +from config import ME +from config import ID +from config import DOMAIN +from config import USERNAME +from config import BASE_URL +from config import ACTOR_SERVICE +from config import OBJECT_SERVICE +from config import PASS +from config import HEADERS +from utils.httpsig import HTTPSigAuth, verify_request +from utils.key import get_secret_key +from utils.webfinger import get_remote_follow_template +from utils.webfinger import get_actor_url + + +app = Flask(__name__) +app.secret_key = get_secret_key('flask') + +JWT_SECRET = get_secret_key('jwt') +JWT = JSONWebSignatureSerializer(JWT_SECRET) + +with open('config/jwt_token', 'wb+') as f: + f.write(JWT.dumps({'type': 'admin_token'})) + +SIG_AUTH = HTTPSigAuth(ID+'#main-key', KEY.privkey) + + +def verify_pass(pwd): + return bcrypt.verify(pwd, PASS) + +@app.context_processor +def inject_config(): + return dict(config=config, logged_in=session.get('logged_in', False)) + +@app.after_request +def set_x_powered_by(response): + response.headers['X-Powered-By'] = 'microblog.pub' + return response + +# HTML/templates helper +ALLOWED_TAGS = [ + 'a', + 'abbr', + 'acronym', + 'b', + 'blockquote', + 'code', + 'pre', + 'em', + 'i', + 'li', + 'ol', + 'strong', + 'ul', + 'span', + 'div', + 'p', + 'h1', + 'h2', + 'h3', + 'h4', + 'h5', + 'h6', +] + + +def clean_html(html): + return bleach.clean(html, tags=ALLOWED_TAGS) + + +@app.template_filter() +def quote_plus(t): + return urllib.parse.quote_plus(t) + + +@app.template_filter() +def clean(html): + return clean_html(html) + + +@app.template_filter() +def html2plaintext(body): + return html2text(body) + + +@app.template_filter() +def domain(url): + return urlparse(url).netloc + + +@app.template_filter() +def get_actor(url): + if not url: + return None + print(f'GET_ACTOR {url}') + return ACTOR_SERVICE.get(url) + +@app.template_filter() +def format_time(val): + if val: + return datetime.strftime(datetime.strptime(val, '%Y-%m-%dT%H:%M:%SZ'), '%B %d, %Y, %H:%M %p') + return val + + +@app.template_filter() +def format_timeago(val): + if val: + try: + return timeago.format(datetime.strptime(val, '%Y-%m-%dT%H:%M:%SZ'), datetime.utcnow()) + except: + return timeago.format(datetime.strptime(val, '%Y-%m-%dT%H:%M:%S.%fZ'), datetime.utcnow()) + + return val + +def _is_img(filename): + filename = filename.lower() + if (filename.endswith('.png') or filename.endswith('.jpg') or filename.endswith('.jpeg') or + filename.endswith('.gif') or filename.endswith('.svg')): + return True + return False + +@app.template_filter() +def not_only_imgs(attachment): + for a in attachment: + if not _is_img(a['url']): + return True + return False + +@app.template_filter() +def is_img(filename): + return _is_img(filename) + + +def login_required(f): + @wraps(f) + def decorated_function(*args, **kwargs): + if not session.get('logged_in'): + return redirect(url_for('login', next=request.url)) + return f(*args, **kwargs) + return decorated_function + + +def api_required(f): + @wraps(f) + def decorated_function(*args, **kwargs): + if session.get('logged_in'): + return f(*args, **kwargs) + + # Token verification + token = request.headers.get('Authorization', '').replace('Bearer ', '') + if not token: + token = request.form.get('access_token', '') + + try: + payload = JWT.loads(token) + # TODO(tsileo): log payload + except BadSignature: + abort(401) + return f(*args, **kwargs) + return decorated_function + + +def jsonify(**data): + if '@context' not in data: + data['@context'] = config.CTX_AS + return Response( + response=json.dumps(data), + headers={'Content-Type': 'application/json' if app.debug else 'application/activity+json'}, + ) + + +def is_api_request(): + h = request.headers.get('Accept') + if h is None: + return False + h = h.split(',')[0] + if h in HEADERS or h == 'application/json': + return True + return False + +# App routes + +####### +# Login + +@app.route('/logout') +@login_required +def logout(): + session['logged_in'] = False + return redirect('/') + + +@app.route('/login', methods=['POST', 'GET']) +def login(): + devices = [doc['device'] for doc in DB.u2f.find()] + u2f_enabled = True if devices else False + if request.method == 'POST': + pwd = request.form.get('pass') + if pwd and verify_pass(pwd): + if devices: + resp = json.loads(request.form.get('resp')) + print(resp) + try: + u2f.complete_authentication(session['challenge'], resp) + except ValueError as exc: + print('failed', exc) + abort(401) + return + finally: + session['challenge'] = None + + session['logged_in'] = True + return redirect(request.args.get('redirect') or '/admin') + else: + abort(401) + + payload = None + if devices: + payload = u2f.begin_authentication(ID, devices) + session['challenge'] = payload + + return render_template( + 'login.html', + u2f_enabled=u2f_enabled, + me=ME, + payload=payload, + ) + + +@app.route('/remote_follow', methods=['GET', 'POST']) +@login_required +def remote_follow(): + if request.method == 'GET': + return render_template('remote_follow.html') + + return redirect(get_remote_follow_template('@'+request.form.get('profile')).format(uri=ID)) + + +@app.route('/authorize_follow', methods=['GET', 'POST']) +@login_required +def authorize_follow(): + if request.method == 'GET': + return render_template('authorize_remote_follow.html', profile=request.args.get('profile')) + + actor = get_actor_url(request.form.get('profile')) + if not actor: + abort(500) + if DB.following.find({'remote_actor': actor}).count() > 0: + return redirect('/following') + + follow = activitypub.Follow(object=actor) + follow.post_to_outbox() + return redirect('/following') + + +@app.route('/u2f/register', methods=['GET', 'POST']) +@login_required +def u2f_register(): + # TODO(tsileo): ensure no duplicates + if request.method == 'GET': + payload = u2f.begin_registration(ID) + session['challenge'] = payload + return render_template( + 'u2f.html', + payload=payload, + ) + else: + resp = json.loads(request.form.get('resp')) + device, device_cert = u2f.complete_registration(session['challenge'], resp) + session['challenge'] = None + DB.u2f.insert_one({'device': device, 'cert': device_cert}) + return '' + +####### +# Activity pub routes + +@app.route('/') +def index(): + print(request.headers.get('Accept')) + if is_api_request(): + return jsonify(**ME) + + # FIXME(tsileo): implements pagination, also for the followers/following page + limit = 50 + q = { + 'type': 'Create', + 'activity.object.type': 'Note', + 'meta.deleted': False, + } + c = request.args.get('cursor') + if c: + q['_id'] = {'$lt': ObjectId(c)} + + outbox_data = list(DB.outbox.find({'$or': [q, {'type': 'Announce', 'meta.undo': False}]}, limit=limit).sort('_id', -1)) + cursor = None + if outbox_data and len(outbox_data) == limit: + cursor = str(outbox_data[-1]['_id']) + + for data in outbox_data: + if data['type'] == 'Announce': + print(data) + if data['activity']['object'].startswith('http'): + data['ref'] = {'activity': {'object': OBJECT_SERVICE.get(data['activity']['object'])}, 'meta': {}} + + + return render_template( + 'index.html', + me=ME, + notes=DB.inbox.find({'type': 'Create', 'activity.object.type': 'Note', 'meta.deleted': False}).count(), + followers=DB.followers.count(), + following=DB.following.count(), + outbox_data=outbox_data, + cursor=cursor, + ) + + +@app.route('/note/') +def note_by_id(note_id): + data = DB.outbox.find_one({'id': note_id, 'meta.deleted': False}) + if not data: + return Response(status=404) + + replies = list(DB.inbox.find({ + 'type': 'Create', + 'activity.object.inReplyTo': data['activity']['object']['id'], + 'meta.deleted': False, + })) + + # Check for "replies of replies" + others = [] + for rep in replies: + for rep_reply in rep.get('meta', {}).get('replies', []): + others.append(rep_reply['id']) + + if others: + # Fetch the latest versions of the "replies of replies" + replies2 = list(DB.inbox.find({ + 'activity.id': {'$in': others}, + })) + + replies.extend(replies2) + + replies2 = list(DB.outbox.find({ + 'activity.id': {'$in': others}, + })) + + replies.extend(replies2) + + + # Re-sort everything + replies = sorted(replies, key=lambda o: o['activity']['object']['published']) + + + return render_template('note.html', me=ME, note=data, replies=replies) + + +@app.route('/.well-known/webfinger') +def webfinger(): + """Enable WebFinger support, required for Mastodon interopability.""" + resource = request.args.get('resource') + if resource not in ["acct:"+USERNAME+"@"+DOMAIN, ID]: + abort(404) + + out = { + "subject": "acct:"+USERNAME+"@"+DOMAIN, + "aliases": [ID], + "links": [ + {"rel": "http://webfinger.net/rel/profile-page", "type": "text/html", "href": BASE_URL}, + {"rel": "self", "type": "application/activity+json", "href": ID}, + {"rel":"http://ostatus.org/schema/1.0/subscribe","template": BASE_URL+"/authorize_follow?profile={uri}"}, + ], + } + + return Response( + response=json.dumps(out), + headers={'Content-Type': 'application/jrd+json; charset=utf-8' if not app.debug else 'application/json'}, + ) + +@app.route('/outbox', methods=['GET', 'POST']) +def outbox(): + if request.method == 'GET': + if not is_api_request(): + abort(404) + # TODO(tsileo): filter the outbox if not authenticated + # FIXME(tsileo): filter deleted, add query support for build_ordered_collection + q = { + 'meta.deleted': False, + 'type': {'$in': [ActivityTypes.CREATE.value, ActivityTypes.ANNOUNCE.value]}, + } + return jsonify(**activitypub.build_ordered_collection( + DB.outbox, + q=q, + cursor=request.args.get('cursor'), + map_func=lambda doc: clean_activity(doc['activity']), + )) + + # Handle POST request + # FIXME(tsileo): check auth + data = request.get_json(force=True) + print(data) + activity = activitypub.parse_activity(data) + + if activity.type_enum == ActivityTypes.NOTE: + activity = activity.build_create() + + activity.post_to_outbox() + + return Response(status=201, headers={'Location': activity.id}) + + +@app.route('/outbox/') +def outbox_detail(item_id): + doc = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + return jsonify(**clean_activity(doc['activity'])) + + +@app.route('/outbox//activity') +def outbox_activity(item_id): + data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + if not data: + abort(404) + obj = data['activity'] + if obj['type'] != ActivityTypes.CREATE.value: + abort(404) + return jsonify(**clean_activity(obj['object'])) + +@app.route('/admin', methods=['GET']) +@login_required +def admin(): + q = { + 'meta.deleted': False, + 'meta.undo': False, + 'type': ActivityTypes.LIKE.value, + } + col_liked = DB.outbox.count(q) + + return render_template( + 'admin.html', + instances=list(DB.instances.find()), + inbox_size=DB.inbox.count(), + outbox_size=DB.outbox.count(), + object_cache_size=DB.objects_cache.count(), + actor_cache_size=DB.actors_cache.count(), + col_liked=col_liked, + col_followers=DB.followers.count(), + col_following=DB.following.count(), + ) + + +@app.route('/new', methods=['GET', 'POST']) +@login_required +def new(): + if request.method == 'POST': + reply = None + if request.form.get('reply'): + reply = activitypub.parse_activity(OBJECT_SERVICE.get(request.form.get('reply'))) + source = request.form.get('content') + content, tags = parse_markdown(source) + to = request.form.get('to') + cc = [ID+'/followers'] + if reply: + cc.append(reply.attributedTo) + for tag in tags: + if tag['type'] == 'Mention': + cc.append(tag['href']) + + note = activitypub.Note( + cc=cc, + to=[to if to else config.AS_PUBLIC], + content=content, # TODO(tsileo): handle markdown + tag=tags, + source={'mediaType': 'text/markdown', 'content': source}, + inReplyTo=reply.id, + ) + create = note.build_create() + print(create.to_dict()) + create.post_to_outbox() + + reply_id = None + content = '' + if request.args.get('reply'): + reply = activitypub.parse_activity(OBJECT_SERVICE.get(request.args.get('reply'))) + reply_id = reply.id + actor = reply.get_actor() + domain = urlparse(actor.id).netloc + content = f'@{actor.preferredUsername}@{domain} ' + + return render_template('new.html', reply=reply_id, content=content) + + +@app.route('/notifications') +@login_required +def notifications(): + # FIXME(tsileo): implements pagination, also for the followers/following page + limit = 50 + q = { + 'type': 'Create', + 'activity.object.tag.type': 'Mention', + 'activity.object.tag.name': f'@{USERNAME}@{DOMAIN}', + 'meta.deleted': False, + } + # TODO(tsileo): also include replies via regex on Create replyTo + q = {'$or': [q, {'type': 'Follow'}, {'type': 'Accept'}, {'type': 'Undo', 'activity.object.type': 'Follow'}, + {'type': 'Announce', 'activity.object': {'$regex': f'^{BASE_URL}'}}, + {'type': 'Create', 'activity.object.inReplyTo': {'$regex': f'^{BASE_URL}'}}, + ]} + print(q) + c = request.args.get('cursor') + if c: + q['_id'] = {'$lt': ObjectId(c)} + + outbox_data = list(DB.inbox.find(q, limit=limit).sort('_id', -1)) + cursor = None + if outbox_data and len(outbox_data) == limit: + cursor = str(outbox_data[-1]['_id']) + + # TODO(tsileo): fix the annonce handling, copy it from /stream + #for data in outbox_data: + # if data['type'] == 'Announce': + # print(data) + # if data['activity']['object'].startswith('http') and data['activity']['object'] in objcache: + # data['ref'] = {'activity': {'object': objcache[data['activity']['object']]}, 'meta': {}} + # out.append(data) + # else: + # out.append(data) + + return render_template( + 'stream.html', + inbox_data=outbox_data, + cursor=cursor, + ) + +@app.route('/ui/boost') +@login_required +def ui_boost(): + oid = request.args.get('id') + obj = activitypub.parse_activity(OBJECT_SERVICE.get(oid)) + announce = obj.build_announce() + announce.post_to_outbox() + return redirect(request.args.get('redirect')) + +@app.route('/ui/like') +@login_required +def ui_like(): + oid = request.args.get('id') + obj = activitypub.parse_activity(OBJECT_SERVICE.get(oid)) + like = obj.build_like() + like.post_to_outbox() + return redirect(request.args.get('redirect')) + +@app.route('/ui/undo') +@login_required +def ui_undo(): + oid = request.args.get('id') + doc =DB.outbox.find_one({'id': oid}) + if doc: + obj = activitypub.parse_activity(doc.get('activity')) + undo = obj.build_undo() + undo.post_to_outbox() + return redirect(request.args.get('redirect')) + +@app.route('/stream') +@login_required +def stream(): + # FIXME(tsileo): implements pagination, also for the followers/following page + limit = 100 + q = { + 'type': 'Create', + 'activity.object.type': 'Note', + 'activity.object.inReplyTo': None, + 'meta.deleted': False, + } + c = request.args.get('cursor') + if c: + q['_id'] = {'$lt': ObjectId(c)} + + outbox_data = list(DB.inbox.find( + { + '$or': [ + q, + { + 'type': 'Announce', + }, + ] + }, limit=limit).sort('activity.published', -1)) + cursor = None + if outbox_data and len(outbox_data) == limit: + cursor = str(outbox_data[-1]['_id']) + + out = [] + objcache = {} + cached = list(DB.objects_cache.find({'meta.part_of_stream': True}, limit=limit*3).sort('meta.announce_published', -1)) + for c in cached: + objcache[c['object_id']] = c['cached_object'] + for data in outbox_data: + if data['type'] == 'Announce': + if data['activity']['object'].startswith('http') and data['activity']['object'] in objcache: + data['ref'] = {'activity': {'object': objcache[data['activity']['object']]}, 'meta': {}} + out.append(data) + else: + print('OMG', data) + else: + out.append(data) + return render_template( + 'stream.html', + inbox_data=out, + cursor=cursor, + ) + + +@app.route('/inbox', methods=['GET', 'POST']) +def inbox(): + if request.method == 'GET': + if not is_api_request(): + abort(404) + # TODO(tsileo): handle auth and only return 404 if unauthenticated + # abort(404) + return jsonify(**activitypub.build_ordered_collection( + DB.inbox, + q={'meta.deleted': False}, + cursor=request.args.get('cursor'), + map_func=lambda doc: doc['activity'], + )) + data = request.get_json(force=True) + # FIXME(tsileo): ensure verify_request() == True + print(data) + try: + print(verify_request(ACTOR_SERVICE)) + except Exception: + print('failed to verify request') + + activity = activitypub.parse_activity(data) + print(activity) + activity.process_from_inbox() + + return Response( + status=201, + ) + + +@app.route('/api/upload', methods=['POST']) +@api_required +def api_upload(): + file = request.files['file'] + rfilename = secure_filename(file.filename) + prefix = hashlib.sha256(os.urandom(32)).hexdigest()[:6] + mtype = mimetypes.guess_type(rfilename)[0] + filename = f'{prefix}_{rfilename}' + file.save(os.path.join('static', 'media', filename)) + + # Remove EXIF metadata + if filename.lower().endswith('.jpg') or filename.lower().endswith('.jpeg'): + piexif.remove(os.path.join('static', 'media', filename)) + + print('upload OK') + print(filename) + attachment = [ + {'mediaType': mtype, + 'name': rfilename, + 'type': 'Document', + 'url': BASE_URL + f'/static/media/{filename}' + }, + ] + print(attachment) + content = request.args.get('content') + to = request.args.get('to') + note = activitypub.Note( + cc=[ID+'/followers'], + to=[to if to else config.AS_PUBLIC], + content=content, # TODO(tsileo): handle markdown + attachment=attachment, + ) + print('post_note_init') + print(note) + create = note.build_create() + print(create) + print(create.to_dict()) + create.post_to_outbox() + print('posted') + + return Response( + status=201, + response='OK', + ) + + +@app.route('/api/new_note') +@api_required +def api_new_note(): + source = request.args.get('content') + content, tags = parse_markdown(source) + to = request.args.get('to') + cc = [ID+'/followers'] + for tag in tags: + if tag['type'] == 'Mention': + cc.append(tag['href']) + + note = activitypub.Note( + cc=cc, + to=[to if to else config.AS_PUBLIC], + content=content, # TODO(tsileo): handle markdown + tag=tags, + source={'mediaType': 'text/markdown', 'content': source}, + ) + create = note.build_create() + create.post_to_outbox() + return Response( + status=201, + response='OK', + ) + +@app.route('/api/stream') +def api_stream(): + return Response( + response=json.dumps(activitypub.build_inbox_json_feed('/api/stream', request.args.get('cursor'))), + headers={'Content-Type': 'application/json'}, + ) + +@app.route('/api/follow') +@api_required +def api_follow(): + actor = request.args.get('actor') + if DB.following.find({'remote_actor': actor}).count() > 0: + return Response(status=201) + + follow = activitypub.Follow(object=actor) + follow.post_to_outbox() + return Response( + status=201, + ) + + +@app.route('/followers') +def followers(): + if is_api_request(): + return jsonify( + **activitypub.build_ordered_collection( + DB.followers, + cursor=request.args.get('cursor'), + map_func=lambda doc: doc['remote_actor'], + ) + ) + + followers = [ACTOR_SERVICE.get(doc['remote_actor']) for doc in DB.followers.find(limit=50)] + return render_template( + 'followers.html', + me=ME, + notes=DB.inbox.find({'object.object.type': 'Note'}).count(), + followers=DB.followers.count(), + following=DB.following.count(), + followers_data=followers, + ) + + +@app.route('/following') +def following(): + if is_api_request(): + return jsonify( + **activitypub.build_ordered_collection( + DB.following, + cursor=request.args.get('cursor'), + map_func=lambda doc: doc['remote_actor'], + ), + ) + + following = [ACTOR_SERVICE.get(doc['remote_actor']) for doc in DB.following.find(limit=50)] + return render_template( + 'following.html', + me=ME, + notes=DB.inbox.find({'object.object.type': 'Note'}).count(), + followers=DB.followers.count(), + following=DB.following.count(), + following_data=following, + ) + + +@app.route('/tags/') +def tags(tag): + if not DB.outbox.count({'activity.object.tag.type': 'Hashtag', 'activity.object.tag.name': '#'+tag}): + abort(404) + if not is_api_request(): + return render_template( + 'tags.html', + tag=tag, + outbox_data=DB.outbox.find({'type': 'Create', 'activity.object.type': 'Note', 'meta.deleted': False, + 'activity.object.tag.type': 'Hashtag', + 'activity.object.tag.name': '#'+tag}), + ) + q = { + 'meta.deleted': False, + 'meta.undo': False, + 'type': ActivityTypes.CREATE.value, + 'activity.object.tag.type': 'Hashtag', + 'activity.object.tag.name': '#'+tag, + } + return jsonify(**activitypub.build_ordered_collection( + DB.outbox, + q=q, + cursor=request.args.get('cursor'), + map_func=lambda doc: doc['activity']['object']['id'], + col_name=f'tags/{tag}', + )) + + +@app.route('/liked') +def liked(): + if not is_api_request(): + abort(404) + q = { + 'meta.deleted': False, + 'meta.undo': False, + 'type': ActivityTypes.LIKE.value, + } + return jsonify(**activitypub.build_ordered_collection( + DB.outbox, + q=q, + cursor=request.args.get('cursor'), + map_func=lambda doc: doc['activity']['object'], + col_name='liked', + )) + +####### +# IndieAuth + + +def build_auth_resp(payload): + if request.headers.get('Accept') == 'application/json': + return Response( + status=200, + headers={'Content-Type': 'application/json'}, + response=json.dumps(payload), + ) + return Response( + status=200, + headers={'Content-Type': 'application/x-www-form-urlencoded'}, + response=urlencode(payload), + ) + + +def _get_prop(props, name, default=None): + if name in props: + items = props.get(name) + if isinstance(items, list): + return items[0] + return items + return default + +def get_client_id_data(url): + data = mf2py.parse(url=url) + for item in data['items']: + if 'h-x-app' in item['type'] or 'h-app' in item['type']: + props = item.get('properties', {}) + print(props) + return dict( + logo=_get_prop(props, 'logo'), + name=_get_prop(props, 'name'), + url=_get_prop(props, 'url'), + ) + return dict( + logo=None, + name=url, + url=url, + ) + + +@app.route('/indieauth/flow', methods=['POST']) +@login_required +def indieauth_flow(): + auth = dict( + scope=' '.join(request.form.getlist('scopes')), + me=request.form.get('me'), + client_id=request.form.get('client_id'), + state=request.form.get('state'), + redirect_uri=request.form.get('redirect_uri'), + response_type=request.form.get('response_type'), + ) + + code = binascii.hexlify(os.urandom(8)).decode('utf-8') + auth.update( + code=code, + verified=False, + ) + print(auth) + if not auth['redirect_uri']: + abort(500) + + DB.indieauth.insert_one(auth) + + # FIXME(tsileo): fetch client ID and validate redirect_uri + red = f'{auth["redirect_uri"]}?code={code}&state={auth["state"]}&me={auth["me"]}' + return redirect(red) + + +@app.route('/indieauth', methods=['GET', 'POST']) +def indieauth_endpoint(): + session['logged_in'] = True + if request.method == 'GET': + if not session.get('logged_in'): + return redirect(url_for('login', next=request.url)) + + me = request.args.get('me') + # FIXME(tsileo): ensure me == ID + client_id = request.args.get('client_id') + redirect_uri = request.args.get('redirect_uri') + state = request.args.get('state', '') + response_type = request.args.get('response_type', 'id') + scope = request.args.get('scope', '').split() + + print('STATE', state) + return render_template( + 'indieauth_flow.html', + client=get_client_id_data(client_id), + scopes=scope, + redirect_uri=redirect_uri, + state=state, + response_type=response_type, + client_id=client_id, + me=me, + ) + + # Auth verification via POST + code = request.form.get('code') + redirect_uri = request.form.get('redirect_uri') + client_id = request.form.get('client_id') + + auth = DB.indieauth.find_one_and_update( + {'code': code, 'redirect_uri': redirect_uri, 'client_id': client_id}, #}, # , 'verified': False}, + {'$set': {'verified': True}}, + sort=[('_id', pymongo.DESCENDING)], + ) + print(auth) + print(code, redirect_uri, client_id) + + if not auth: + abort(403) + return + + me = auth['me'] + state = auth['state'] + scope = ' '.join(auth['scope']) + print('STATE', state) + return build_auth_resp({'me': me, 'state': state, 'scope': scope}) + + +@app.route('/token', methods=['GET', 'POST']) +def token_endpoint(): + if request.method == 'POST': + code = request.form.get('code') + me = request.form.get('me') + redirect_uri = request.form.get('redirect_uri') + client_id = request.form.get('client_id') + + auth = DB.indieauth.find_one({'code': code, 'me': me, 'redirect_uri': redirect_uri, 'client_id': client_id}) + if not auth: + abort(403) + scope = ' '.join(auth['scope']) + payload = dict(me=me, client_id=client_id, scope=scope, ts=datetime.now().timestamp()) + token = JWT.dumps(payload).decode('utf-8') + + return build_auth_resp({'me': me, 'scope': scope, 'access_token': token}) + + # Token verification + token = request.headers.get('Authorization').replace('Bearer ', '') + try: + payload = JWT.loads(token) + except BadSignature: + abort(403) + + # TODO(tsileo): handle expiration + + return build_auth_resp({ + 'me': payload['me'], + 'scope': payload['scope'], + 'client_id': payload['client_id'], + }) diff --git a/config.py b/config.py new file mode 100644 index 0000000..b8ff84f --- /dev/null +++ b/config.py @@ -0,0 +1,83 @@ +import os +import yaml +from pymongo import MongoClient +import requests + +from utils.key import Key +from utils.actor_service import ActorService +from utils.object_service import ObjectService + + +VERSION = '1.0.0' + +CTX_AS = 'https://www.w3.org/ns/activitystreams' +CTX_SECURITY = 'https://w3id.org/security/v1' +AS_PUBLIC = 'https://www.w3.org/ns/activitystreams#Public' +HEADERS = [ + 'application/activity+json', + 'application/ld+json;profile=https://www.w3.org/ns/activitystreams', + 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', + 'application/ld+json', +] + + +with open('config/me.yml') as f: + conf = yaml.load(f) + + USERNAME = conf['username'] + NAME = conf['name'] + DOMAIN = conf['domain'] + SCHEME = 'https' if conf.get('https', True) else 'http' + BASE_URL = SCHEME + '://' + DOMAIN + ID = BASE_URL + SUMMARY = conf['summary'] + ICON_URL = conf['icon_url'] + PASS = conf['pass'] + PUBLIC_INSTANCES = conf.get('public_instances') + +USER_AGENT = ( + f'{requests.utils.default_user_agent()} ' + f'(microblog.pub/{VERSION}; +{BASE_URL})' +) + +# TODO(tsileo): use 'mongo:27017; +# mongo_client = MongoClient(host=['mongo:27017']) +mongo_client = MongoClient( + host=[os.getenv('MICROBLOGPUB_MONGODB_HOST', 'localhost:27017')], +) + +DB = mongo_client['{}_{}'.format(USERNAME, DOMAIN.replace('.', '_'))] +KEY = Key(USERNAME, DOMAIN, create=True) + +ME = { + "@context": [ + CTX_AS, + CTX_SECURITY, + ], + "type": "Person", + "id": ID, + "following": ID+"/following", + "followers": ID+"/followers", + "liked": ID+"/liked", + "inbox": ID+"/inbox", + "outbox": ID+"/outbox", + "preferredUsername": USERNAME, + "name": NAME, + "summary": SUMMARY, + "endpoints": {}, + "url": ID, + "icon": { + "mediaType": "image/png", + "type": "Image", + "url": ICON_URL, + }, + "publicKey": { + "id": ID+"#main-key", + "owner": ID, + "publicKeyPem": KEY.pubkey_pem, + }, +} +print(ME) + +ACTOR_SERVICE = ActorService(USER_AGENT, DB.actors_cache, ID, ME, DB.instances) +OBJECT_SERVICE = ObjectService(USER_AGENT, DB.objects_cache, DB.inbox, DB.outbox, DB.instances) diff --git a/config/me.sample.yml b/config/me.sample.yml new file mode 100644 index 0000000..56e37b0 --- /dev/null +++ b/config/me.sample.yml @@ -0,0 +1,6 @@ +username: 'username' +name: 'You Name' +icon_url: 'https://you-avatar-url' +domain: 'your-domain.tld' +summary: 'your summary' +https: true diff --git a/data/.gitignore b/data/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/data/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml new file mode 100644 index 0000000..0967722 --- /dev/null +++ b/docker-compose-dev.yml @@ -0,0 +1,21 @@ +version: '3' +services: + celery: + build: . + links: + - mongo + - rabbitmq + command: 'celery worker -l info -A tasks' + environment: + - MICROBLOGPUB_AMQP_BORKER=pyamqp://guest@rabbitmq// + - MICROBLOGPUB_MONGODB_HOST=mongo:27017 + mongo: + image: "mongo:latest" + volumes: + - "./data:/data/db" + ports: + - "27017:27017" + rabbitmq: + image: "rabbitmq:latest" + ports: + - "5672:5672" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..50516aa --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,30 @@ +version: '3' +services: + web: + build: . + ports: + - "5005:5005" + links: + - mongo + - rabbitmq + volumes: + - "./config:/app/config" + - "./static:/app/static" + environment: + - MICROBLOGPUB_AMQP_BORKER=pyamqp://guest@rabbitmq// + - MICROBLOGPUB_MONGODB_HOST=mongo:27017 + celery: + build: . + links: + - mongo + - rabbitmq + command: 'celery worker -l info -A tasks' + environment: + - MICROBLOGPUB_AMQP_BORKER=pyamqp://guest@rabbitmq// + - MICROBLOGPUB_MONGODB_HOST=mongo:27017 + mongo: + image: "mongo:latest" + volumes: + - "./data:/data/db" + rabbitmq: + image: "rabbitmq:latest" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3e770ad --- /dev/null +++ b/requirements.txt @@ -0,0 +1,21 @@ +libsass +gunicorn +piexif +requests +markdown +python-u2flib-server +Flask +Celery +pymongo +pyld +timeago +bleach +pycryptodome +html2text +feedgen +itsdangerous +bcrypt +mf2py +passlib +pyyaml +git+https://github.com/erikriver/opengraph.git diff --git a/sass/base_theme.scss b/sass/base_theme.scss new file mode 100644 index 0000000..8a6fba0 --- /dev/null +++ b/sass/base_theme.scss @@ -0,0 +1,207 @@ +// a4.io dark theme +$background-color: #060606; +$background-light: #222; +$color: #808080; +$color-title-link: #fefefe; +$color-summary: #ddd; +$color-light: #bbb; +$color-menu-background: #222; +$color-note-link: #666; +$primary-color: #f7ca18; + +$background-color: #eee; +$background-light: #ccc; +$color: #111; +$color-title-link: #333; +$color-light: #555; +$color-summary: #111; +$color-note-link: #333; +$color-menu-background: #ddd; +$primary-color: #1d781d; + +.note-container p:first-child { + margin-top: 0; +} +html, body { + height: 100%; +} +body { + background-color: $background-color; + color: $color; + display: flex; + flex-direction: column; +} +.base-container { + flex: 1 0 auto; +} +.footer { + flex-shrink: 0; +} +a, h1, h2, h3, h4, h5, h6 { + color: $color-title-link; +} +a { + text-decoration: none; +} +a:hover { + text-decoration: underline; +} +.gold { + color: $primary-color; +} + +#header { + margin-bottom: 40px; + + .title { + font-size: 1.2em; + padding-right: 15px; + color: $color-title-link; + } + .title:hover { + text-decoration: none; + } + .subtitle-username { + color: $color; + } + .menu { + padding: 20px 0 10px 0; + ul { + display: inline; + list-style-type: none; + padding: 0; + li { + float:left; + padding-right:10px; + margin-bottom:10px; + } + } + a { + padding: 2px 7px; + } + a.selected { + background: $primary-color; + color: $background-color; + border-radius:2px; + } + a:hover { + background: $primary-color; + color: $background-color; + text-decoration: none; + } + } +} +#container { + width: 90%; + max-width: 720px; + margin: 40px auto; +} +#container #notes { + margin-top: 20px; +} +.actor-box { + display: block; + text-decoration: none; + margin-bottom: 40px; + + .actor-icon { + width: 100%; + max-width:120px; + border-radius:2px; + } + h3 { margin: 0; } +} +.note { + display: flex; + margin-bottom: 70px; + .l { + color: $color-note-link; + } + + .h-card { + flex: initial; + width: 50px; + } + + .u-photo { + width: 50px; + border-radius: 2px; + } + .note-wrapper { + flex: 1; + padding-left: 15px; + } + + .bottom-bar { margin-top:10px; } + + .img-attachment { + max-width:100%; + border-radius:2px; + } + + h3 { + font-size: 1.1em; + color: $color-light; + } + + strong { font-weight:600; } + + .note-container { + clear: right; + padding:10px 0; + } +} + +.bar-item { + background: $color-menu-background; + padding: 5px; + color: $color-light; + margin-right:5px; + border-radius:2px; +} + +.bottom-bar .perma-item { + margin-right:5px; +} +.bottom-bar a.bar-item:hover { + text-decoration: none; +} +.footer > div { + width: 90%; + max-width: 720px; + margin: 40px auto; +} +.footer a, .footer a:hover, .footer a:visited { + text-decoration: underline; + color: $color; +} +.summary { + color: $color-summary; + font-size:1.3em; + margin-top:50px; + margin-bottom:70px; +} +.summary a, .summay a:hover { + color: $color-summary; + text-decoration:underline; +} +#followers, #following, #new { + margin-top:50px; +} +#admin { + margin-top:50px; +} +textarea, input { + background: $color-menu-background; + padding: 10px; + color: $color-light; + border: 0px; + border-radius: 2px; +} +input { + padding: 10px; +} +input[type=submit] { + color: $primary-color; + text-transform: uppercase; +} diff --git a/sass/theme.scss b/sass/theme.scss new file mode 100644 index 0000000..6277839 --- /dev/null +++ b/sass/theme.scss @@ -0,0 +1 @@ +@import 'base_theme.scss' diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..6deafc2 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 120 diff --git a/static/base.css b/static/base.css new file mode 100644 index 0000000..aeab1f3 --- /dev/null +++ b/static/base.css @@ -0,0 +1,160 @@ +.note-container p:first-child { + margin-top: 0; +} +html, body { + height: 100%; +} +body { + background-color: #060606; + color: #808080; +display: flex; +flex-direction: column; +} +.base-container { + flex: 1 0 auto; +} +.footer { +flex-shrink: 0; +} +a, h1, h2, h3, h4, h5, h6 { + color: #fefefe; +} +a { + text-decoration: none; +} +a:hover { + text-decoration: underline; +} +.gold { + color: #f7ca18; +} + +#header { + margin-bottom: 40px; +} +#header .h-card { +} +#header .title { +font-size:1.2em;padding-right:15px;color:#fefefe; +} +#header .title:hover { +text-decoration: none; +} +#header .menu { +padding: 20px 0 10px 0; +} +#header .menu ul { +display:inline;list-style-type:none;padding:0; +} +#header .menu li { +float:left; +padding-right:10px; +margin-bottom:10px; +} +#header .menu a { + padding: 2px 7px; +} +#header .menu a.selected { +background:#f7ca18;color:#634806; +border-radius:2px; +} +#header .menu a:hover { +background:#f7ca18;color:#060606; +text-decoration: none; +} +#container { +width:90%;max-width: 720px;margin:40px auto; +} +#container #notes { +margin-top:20px; +} +.actor-box { +display:block;text-decoration:none;margin-bottom:40px; +} +.actor-box .actor-icon { +width: 100%; +max-width:120px; +border-radius:2px; +} +.actor-box h3 { margin:0; } +.note .l { +color:#666; +} +.note { +display:flex;margin-bottom:70px; +} +.note .h-card { +flex:initial;width:50px; +} +.note .u-photo { +width:50px;border-radius:2px; +} +.note .note-wrapper { +flex:1;padding-left:15px; +} +.note .bottom-bar { +margin-top:10px; +} +.bar-item { +background: #222; +padding: 5px; +color:#bbb; +margin-right:5px; +border-radius:2px; +} +.bottom-bar .perma-item { +margin-right:5px; +} +.bottom-bar a.bar-item:hover { + text-decoration: none; +} +.note .img-attachment { +max-width:100%; +border-radius:2px; +} +.note h3 { +font-size:1.1em;color:#ccc; +} +.note .note-container { +clear:right;padding:10px 0; +} +.note strong { +font-weight:600; +} +.footer > div { +width:90%;max-width: 720px;margin:40px auto; +} +.footer a, .footer a:hover, .footer a:visited { + text-decoration:underline; + color:#808080; +} +.summary { +color: #ddd; +font-size:1.3em; +margin-top:50px; +margin-bottom:70px; +} +.summary a, .summay a:hover { +color:#ddd; +text-decoration:underline; +} +#followers, #following, #new { + margin-top:50px; +} +#admin { + margin-top:50px; +} +textarea, input { + background: #222; + padding: 10px; + color: #bbb; + border: 0px; + border-radius: 2px; +} +input { + padding: 10px; +} +input[type=submit] { + color: #f7ca18; + text-transform: uppercase; +} diff --git a/static/css/.gitignore b/static/css/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/static/css/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/static/media/.gitignore b/static/media/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/static/media/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/static/nopic.png b/static/nopic.png new file mode 100644 index 0000000000000000000000000000000000000000..988d8066e0ac3bb640f6d97a83ee3ad0d0d39d06 GIT binary patch literal 1791 zcma)-2{e>z7{{l{GD(OIvNc7KNJlO$gGrdiWQ?sbEeI3Jl4XRc8I!KL(pXZK$r=*I zIw2H=NF!^OYpJfG;o9yP#-97lx#x7%>7H}H?|r}becto@|KD@Y^PS%pi9sWFh$)DH zK%gDSGp07c`c^O@BEYlIp(qVlL|jk^Q_zOs&Ul#{51eefeg+o+0&SNO%q<{#iac;A z8i=$s6CD8yOP>$}CGo((k+P?qL*Qv&Z*RO$AP|B;r~UDl1MzOq8=iq4Pz2Hv<8~M? z3cSG;X?n^ocwlhm_?8`QaZRJCY`Hyry^*7h@nqlqD3y&;6Qq>Xd ze(up8di0%P8+Nb6K;SJ9A*fUseDekl1yMjkU@6G=n;+8z!k?rGgqz*I`G4>B%^xb| zd_0BWOnAfgqs7EbcIR5r)6#l}hEl$8IOQcJ8R^Dk%~UjNI;fze1T$Fai?z0nvp}Kj z9UY6{P-x}gdueQ(ssinbL&H0PumD>;|8<>cdG@SE{Bq61lc{KXd;5tcE;~OjFAk5> zN`%kZYCi1x{5eBJL}Y%M$4RLp`T3Qi(~SxvB194_q06etXEmiGHGmO!hOx9dJy>bQ zX;1HKOFfA*qA;9LD3pS{JcdlJBdsl!=Fd)ZgLg?#%gGxg2_Xu@n9XMYIWyDy;ll^| zx_AOUudvXPDxo|QHY$}E5|F`U23WFVvz_Wl69eVf?!IwRk^;X@C%vB>AMaROgR$Q3 zoC#;}>r>Iw!!=l-a7}G(dn7XQg=6uMPv!lV>=EwsEAH;U=<7eZG5b+_wRKlab5D17 z7M~v^s64B>`aB|9$=cg{dXFzZ9NWy4Ur*Up&z`}dO<6ew_bqK?x!HR;Zm#Kn8Qy8OjO83=`8B|$yCu&{8| z)6*Kb(2hFt;_o=TJSMYS`bN#-#bsqS zx#(0IKR@o>xGsUAIVfum<+FLq_r5^xz`a^V#phN>wm=$#YRMaEgr+hrqsJ&HpEt&m z)7H{*OC5YkG+^zMS`ArWDet^!W+n^-MBMrFUBG8ayK?R7RR?)WKPz6{O3%O`k>JBh z(7q{@2%o|S1)Y;rfgPcHdwJ>Ab_s)9h*42d)ui=mSvfgo+saKN;qX%lKq1RD>?WEb zJt^r3-I&N{==*+b9f35e_zk@Xu~k)7HRlls1lZYV_>>F+i%opo`LuJb=u*vxfq{4b zq}ATKpyO1lM_&wvRR>Pohp9-iRAXt{@hwklYA%F_hnw@zxwdjTjt@S~x`zmraoNuU zIMHdwq6prJC-`M8v4qeX*HvEM|= zSsaMRaqmiMf5mYL(?fJ#Sznbhlt|TF7Uz|nGJeS?f2xyL0X0`OGhr#FG zjgDq_9u1hBoW#je`rBJu?{yBP84?|UAg09@S>Zj{W4jg~~DwaH)Fy zNzNmEZx?GJ6DZjQCmRLyR=4W5J?2&B-hUAYAT8;-X_j@Zr^;vX%@B;O1TeL5smkAs z8w{Jgh(L(4Mm8mU&(OK8d#Z$wh5v+)Kf@@X{gJl+iBX8q|EBX#Y!zFL>n=ajOd)yn Q3;sh$Gqh=uiSzBh0kk;&aR2}S literal 0 HcmV?d00001 diff --git a/tasks.py b/tasks.py new file mode 100644 index 0000000..7079491 --- /dev/null +++ b/tasks.py @@ -0,0 +1,56 @@ +import os +import logging +import random + +import requests +from celery import Celery +from requests.exceptions import HTTPError + +from config import HEADERS +from config import ID +from config import DB +from config import KEY +from config import USER_AGENT +from utils.httpsig import HTTPSigAuth +from utils.opengraph import fetch_og_metadata + + +log = logging.getLogger() +app = Celery('tasks', broker=os.getenv('MICROBLOGPUB_AMQP_BROKER', 'pyamqp://guest@localhost//')) +# app = Celery('tasks', broker='pyamqp://guest@rabbitmq//') +SigAuth = HTTPSigAuth(ID+'#main-key', KEY.privkey) + + +@app.task(bind=True, max_retries=12) +def post_to_inbox(self, payload, to): + try: + log.info('payload=%s', payload) + log.info('to=%s', to) + resp = requests.post(to, data=payload, auth=SigAuth, headers={ + 'Content-Type': HEADERS[1], + 'Accept': HEADERS[1], + 'User-Agent': USER_AGENT, + }) + print(resp) + log.info('resp=%s', resp) + log.info('resp_body=%s', resp.text) + resp.raise_for_status() + except HTTPError as err: + log.exception('request failed') + if 400 >= err.response.status_code >= 499: + log.info('client error, no retry') + return + self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) + + +@app.task(bind=True, max_retries=12) +def fetch_og(self, col, remote_id): + try: + log.info('fetch_og_meta remote_id=%s col=%s', remote_id, col) + if col == 'INBOX': + log.info('%d links saved', fetch_og_metadata(USER_AGENT, DB.inbox, remote_id)) + elif col == 'OUTBOX': + log.info('%d links saved', fetch_og_metadata(USER_AGENT, DB.outbox, remote_id)) + except Exception as err: + self.log.exception('failed') + self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) diff --git a/templates/about.html b/templates/about.html new file mode 100644 index 0000000..7625840 --- /dev/null +++ b/templates/about.html @@ -0,0 +1,22 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} +{% endblock %} +{% block content %} +
+ +{% include "header.html" %} + +
+
+ +
+ +
+{{ text | safe }} +
{{ me.summary }}
+
+
+ +
+{% endblock %} diff --git a/templates/admin.html b/templates/admin.html new file mode 100644 index 0000000..90e11c7 --- /dev/null +++ b/templates/admin.html @@ -0,0 +1,30 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block content %} +
+{% include "header.html" %} +
+

Stats

+

DB

+
    +
  • Inbox size: {{ inbox_size }}
    • +
  • Outbox size: {{ outbox_size }}
    • +
  • Object cache size: {{ object_cache_size }}
    • +
  • Actor cache size: {{ actor_cache_size }}
    • +
+

Collections

+
    +
  • followers: {{ col_followers }}
    • +
  • following: {{ col_following }}
    • +
  • liked: {{col_liked }}
    • +
+

Known Instances

+ +
+ +
+{% endblock %} diff --git a/templates/authorize_remote_follow.html b/templates/authorize_remote_follow.html new file mode 100644 index 0000000..78b7ef0 --- /dev/null +++ b/templates/authorize_remote_follow.html @@ -0,0 +1,15 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} +{% endblock %} +{% block content %} +
+

You're about to follow {{ profile}}

+ +
+ + +
+ +
+{% endblock %} diff --git a/templates/followers.html b/templates/followers.html new file mode 100644 index 0000000..c4d03e2 --- /dev/null +++ b/templates/followers.html @@ -0,0 +1,16 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} +{% endblock %} +{% block content %} +
+{% include "header.html" %} + +
+ {% for follower in followers_data %} + {{ utils.display_actor(follower) }} + {% endfor %} +
+ +
+{% endblock %} diff --git a/templates/following.html b/templates/following.html new file mode 100644 index 0000000..c783133 --- /dev/null +++ b/templates/following.html @@ -0,0 +1,16 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} +{% endblock %} +{% block content %} +
+{% include "header.html" %} + +
+ {% for followed in following_data %} + {{ utils.display_actor(followed) }} + {% endfor %} +
+ +
+{% endblock %} diff --git a/templates/header.html b/templates/header.html new file mode 100644 index 0000000..e3ca067 --- /dev/null +++ b/templates/header.html @@ -0,0 +1,29 @@ + diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..acf6b16 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,45 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} + + + + + + + + + + + + +{% endblock %} +{% block content %} +
+ + +{% include "header.html" %} + +
+{{ config.SUMMARY | safe }} +
+ +
+ {% for item in outbox_data %} + + {% if item.type == 'Announce' %} + {% set boost_actor = item.activity.actor | get_actor %} +

{{ boost_actor.name }} boosted

+ {% if item.ref %} + {{ utils.display_note(item.ref, ui=False) }} + {% endif %} + + {% elif item.type == 'Create' %} + {{ utils.display_note(item) }} + {% endif %} + + {% endfor %} +
+ +
+{% endblock %} diff --git a/templates/indieauth_flow.html b/templates/indieauth_flow.html new file mode 100644 index 0000000..c0ce2c9 --- /dev/null +++ b/templates/indieauth_flow.html @@ -0,0 +1,42 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} +{% endblock %} +{% block content %} +
+ {% if session.logged_in %}logged{% else%}not logged{%endif%} + +
+{% if client.logo %} +
+ +
+{% endif %} +
+
+{{ client.name }} +

wants you to login

+
+
+
+ +
+ {% if scopes %} +

Scopes

+
    + {% for scope in scopes %} +
  • +
    • + {% endfor %} +
+ {% endif %} + + + + + + +
+ +
+{% endblock %} diff --git a/templates/layout.html b/templates/layout.html new file mode 100644 index 0000000..a50d6fb --- /dev/null +++ b/templates/layout.html @@ -0,0 +1,29 @@ + + + + + + +{{ config.USERNAME }} + + + + + + + + + + + + +
+{% block content %}{% endblock %} +
+
+
+ Powered by microblog.pub (source code) and the ActivityPub protocol +
+
+ + diff --git a/templates/login.html b/templates/login.html new file mode 100644 index 0000000..2c16998 --- /dev/null +++ b/templates/login.html @@ -0,0 +1,36 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} +{% endblock %} +{% block content %} +
+ {% if session.logged_in %}logged{% else%}not logged{%endif%} + +
+ + {% if u2f_enabled %} + + + {% else %} + + {% endif %} +
+ +
+{% if u2f_enabled %} + +{% endif %} +{% endblock %} diff --git a/templates/new.html b/templates/new.html new file mode 100644 index 0000000..2e73f38 --- /dev/null +++ b/templates/new.html @@ -0,0 +1,17 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block content %} +
+{% include "header.html" %} +
+
+{% if reply %}{% endif %} + +
+ +
+
+ +
+
+{% endblock %} diff --git a/templates/note.html b/templates/note.html new file mode 100644 index 0000000..55bdfa2 --- /dev/null +++ b/templates/note.html @@ -0,0 +1,23 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} + + + + + + + + + + +{% endblock %} +{% block content %} +
+{% include "header.html" %} +{{ utils.display_note(note, perma=True) }} +{% for reply in replies %} +{{ utils.display_note(reply, perma=False) }} +{% endfor %} +
+{% endblock %} diff --git a/templates/remote_follow.html b/templates/remote_follow.html new file mode 100644 index 0000000..8b54d49 --- /dev/null +++ b/templates/remote_follow.html @@ -0,0 +1,16 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} +{% endblock %} +{% block content %} +
+{% include "header.html" %} +

You're about to follow me

+ +
+ + +
+ +
+{% endblock %} diff --git a/templates/stream.html b/templates/stream.html new file mode 100644 index 0000000..4759ae0 --- /dev/null +++ b/templates/stream.html @@ -0,0 +1,39 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block content %} +
+{% include "header.html" %} +
+ +
+ {% for item in inbox_data %} + {% if item.type == 'Create' %} + {{ utils.display_note(item, ui=True) }} + {% else %} + + {% if item.type == 'Announce' %} + + {% set boost_actor = item.activity.actor | get_actor %} +

{{ boost_actor.name or boost_actor.preferredUsername }} boosted

+ {% if item.ref %} + {{ utils.display_note(item.ref, ui=True) }} + {% endif %} + {% endif %} + + {% if item.type == 'Follow' %} +

{{ item.activity.actor }} followed you

+ {% elif item.type == 'Accept' %} +

you followed {{ item.activity.actor }}

+ {% elif item.type == 'Undo' %} +

{{ item.activity.actor }} unfollowed you

+ {% else %} + {% endif %} + + + {% endif %} + {% endfor %} +
+
+ +
+{% endblock %} diff --git a/templates/tags.html b/templates/tags.html new file mode 100644 index 0000000..fc02452 --- /dev/null +++ b/templates/tags.html @@ -0,0 +1,30 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} + + + + + + + + + + + + +{% endblock %} +{% block content %} +
+ + +{% include "header.html" %} +

#{{ tag }}

+
+ {% for item in outbox_data %} + {{ utils.display_note(item) }} + {% endfor %} +
+ +
+{% endblock %} diff --git a/templates/u2f.html b/templates/u2f.html new file mode 100644 index 0000000..0d94691 --- /dev/null +++ b/templates/u2f.html @@ -0,0 +1,29 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block header %} +{% endblock %} +{% block content %} +
+ {% if session.logged_in %}logged{% else%}not logged{%endif%} + +
+ + +
+ +
+ +{% endblock %} diff --git a/templates/utils.html b/templates/utils.html new file mode 100644 index 0000000..5808374 --- /dev/null +++ b/templates/utils.html @@ -0,0 +1,91 @@ +{% macro display_actor(follower) -%} + +
+
+{% if not follower.icon %} + +{% else %} +{% endif %} +
+
+

{{ follower.name or follower.preferredUsername }}

+@{{ follower.preferredUsername }}@{{ follower.url | domain }} +
{{ follower.summary | safe }}
+
+
+ +{%- endmacro %} + +{% macro display_note(item, perma=False, ui=False) -%} +{% set actor = item.activity.object.attributedTo | get_actor %} +
+ +
+ + + +
+ +
+ {{ actor.name or actor.preferredUsername }} @{{ actor.preferredUsername }}@{{ actor.url | domain }} + + {% if not perma %} + + + + + {% endif %} + {% if item.activity.object.summary %}

{{ item.activity.object.summary }}

{% endif %} +
+ {{ item.activity.object.content | safe }} +
+ + {% if item.activity.object.attachment %} +
+ {% if item.activity.object.attachment | not_only_imgs %} +

Attachment

+ + {% endif %} +
+ {% endif %} + +
+{% if perma %}{{ item.activity.object.published | format_time }} {% endif %} +permalink +{% if item.meta.count_reply %}{{ item.meta.count_reply }} replies{% endif %} +{% if item.meta.count_boost %}{{ item.meta.count_boost }} boosts{% endif %} +{% if item.meta.count_like %}{{ item.meta.count_like }} likes{% endif %} +{% if ui %} +{% set aid = item.activity.object.id | quote_plus %} +reply + +{% set redir = request.path + "#activity-" + item['_id'].__str__() %} + +{% if item.meta.boosted %} +unboost +{% else %} +boost +{% endif %} + +{% if item.meta.liked %} +unlike +{% else %} +like +{% endif %} + +{% endif %} +
+
+ +
+{%- endmacro %} diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/actor_service.py b/utils/actor_service.py new file mode 100644 index 0000000..af7acc8 --- /dev/null +++ b/utils/actor_service.py @@ -0,0 +1,72 @@ +import logging + +import requests +from urllib.parse import urlparse +from Crypto.PublicKey import RSA + +logger = logging.getLogger(__name__) + + +class NotAnActorError(Exception): + def __init__(self, activity): + self.activity = activity + + +class ActorService(object): + def __init__(self, user_agent, col, actor_id, actor_data, instances): + logger.debug(f'Initializing ActorService user_agent={user_agent}') + self._user_agent = user_agent + self._col = col + self._in_mem = {actor_id: actor_data} + self._instances = instances + self._known_instances = set() + + def _fetch(self, actor_url): + logger.debug(f'fetching remote object {actor_url}') + resp = requests.get(actor_url, headers={ + 'Accept': 'application/activity+json', + 'User-Agent': self._user_agent, + }) + resp.raise_for_status() + return resp.json() + + def get(self, actor_url, reload_cache=False): + logger.info(f'get actor {actor_url} (reload_cache={reload_cache})') + + if actor_url in self._in_mem: + return self._in_mem[actor_url] + + instance = urlparse(actor_url)._replace(path='', query='', fragment='').geturl() + if instance not in self._known_instances: + self._known_instances.add(instance) + if not self._instances.find_one({'instance': instance}): + self._instances.insert({'instance': instance, 'first_object': actor_url}) + + if reload_cache: + actor = self._fetch(actor_url) + self._in_mem[actor_url] = actor + self._col.update({'actor_id': actor_url}, {'$set': {'cached_response': actor}}, upsert=True) + return actor + + cached_actor = self._col.find_one({'actor_id': actor_url}) + if cached_actor: + return cached_actor['cached_response'] + + actor = self._fetch(actor_url) + if not 'type' in actor: + raise NotAnActorError(None) + if actor['type'] != 'Person': + raise NotAnActorError(actor) + + self._col.update({'actor_id': actor_url}, {'$set': {'cached_response': actor}}, upsert=True) + self._in_mem[actor_url] = actor + return actor + + def get_public_key(self, actor_url, reload_cache=False): + profile = self.get(actor_url, reload_cache=reload_cache) + pub = profile['publicKey'] + return pub['id'], RSA.importKey(pub['publicKeyPem']) + + def get_inbox_url(self, actor_url, reload_cache=False): + profile = self.get(actor_url, reload_cache=reload_cache) + return profile.get('inbox') diff --git a/utils/content_helper.py b/utils/content_helper.py new file mode 100644 index 0000000..18fabf4 --- /dev/null +++ b/utils/content_helper.py @@ -0,0 +1,57 @@ +import typing +import re + +from bleach.linkifier import Linker +from markdown import markdown + +from utils.webfinger import get_actor_url +from config import USERNAME, BASE_URL, ID +from config import ACTOR_SERVICE + +from typing import List, Optional, Tuple, Dict, Any, Union, Type + + +def set_attrs(attrs, new=False): + attrs[(None, u'target')] = u'_blank' + attrs[(None, u'class')] = u'external' + attrs[(None, u'rel')] = u'noopener' + attrs[(None, u'title')] = attrs[(None, u'href')] + return attrs + + +LINKER = Linker(callbacks=[set_attrs]) +HASHTAG_REGEX = re.compile(r"(#[\d\w\.]+)") +MENTION_REGEX = re.compile(r"@[\d\w_.+-]+@[\d\w-]+\.[\d\w\-.]+") + + +def hashtagify(content: str) -> Tuple[str, List[Dict[str, str]]]: + tags = [] + for hashtag in re.findall(HASHTAG_REGEX, content): + tag = hashtag[1:] + link = f'' + tags.append(dict(href=f'{BASE_URL}/tags/{tag}', name=hashtag, type='Hashtag')) + content = content.replace(hashtag, link) + return content, tags + + +def mentionify(content: str) -> Tuple[str, List[Dict[str, str]]]: + tags = [] + for mention in re.findall(MENTION_REGEX, content): + _, username, domain = mention.split('@') + actor_url = get_actor_url(mention) + p = ACTOR_SERVICE.get(actor_url) + tags.append(dict(type='Mention', href=p['id'], name=mention)) + link = f'@{username}' + content = content.replace(mention, link) + return content, tags + + +def parse_markdown(content: str) -> Tuple[str, List[Dict[str, str]]]: + tags = [] + content = LINKER.linkify(content) + content, hashtag_tags = hashtagify(content) + tags.extend(hashtag_tags) + content, mention_tags = mentionify(content) + tags.extend(mention_tags) + content = markdown(content) + return content, tags diff --git a/utils/httpsig.py b/utils/httpsig.py new file mode 100644 index 0000000..a2e77c5 --- /dev/null +++ b/utils/httpsig.py @@ -0,0 +1,87 @@ +"""Implements HTTP signature for Flask requests. + +Mastodon instances won't accept requests that are not signed using this scheme. + +""" +from datetime import datetime +from urllib.parse import urlparse +from typing import Any, Dict +import base64 +import hashlib + +from flask import request +from requests.auth import AuthBase + +from Crypto.Signature import PKCS1_v1_5 +from Crypto.Hash import SHA256 + + +def _build_signed_string(signed_headers: str, method: str, path: str, headers: Any, body_digest: str) -> str: + out = [] + for signed_header in signed_headers.split(' '): + if signed_header == '(request-target)': + out.append('(request-target): '+method.lower()+' '+path) + elif signed_header == 'digest': + out.append('digest: '+body_digest) + else: + out.append(signed_header+': '+headers[signed_header]) + return '\n'.join(out) + + +def _parse_sig_header(val: str) -> Dict[str, str]: + out = {} + for data in val.split(','): + k, v = data.split('=', 1) + out[k] = v[1:len(v)-1] + return out + + +def _verify_h(signed_string, signature, pubkey): + signer = PKCS1_v1_5.new(pubkey) + digest = SHA256.new() + digest.update(signed_string.encode('utf-8')) + return signer.verify(digest, signature) + + +def _body_digest() -> str: + h = hashlib.new('sha256') + h.update(request.data) + return 'SHA-256='+base64.b64encode(h.digest()).decode('utf-8') + + +def verify_request(actor_service) -> bool: + hsig = _parse_sig_header(request.headers.get('Signature')) + signed_string = _build_signed_string(hsig['headers'], request.method, request.path, request.headers, _body_digest()) + _, rk = actor_service.get_public_key(hsig['keyId']) + return _verify_h(signed_string, base64.b64decode(hsig['signature']), rk) + + +class HTTPSigAuth(AuthBase): + def __init__(self, keyid, privkey): + self.keyid = keyid + self.privkey = privkey + + def __call__(self, r): + host = urlparse(r.url).netloc + bh = hashlib.new('sha256') + bh.update(r.body.encode('utf-8')) + bodydigest = 'SHA-256='+base64.b64encode(bh.digest()).decode('utf-8') + date = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT') + r.headers.update({'Digest': bodydigest, 'Date': date}) + r.headers.update({'Host': host}) + sigheaders = '(request-target) user-agent host date digest content-type' + to_be_signed = _build_signed_string(sigheaders, r.method, r.path_url, r.headers, bodydigest) + signer = PKCS1_v1_5.new(self.privkey) + digest = SHA256.new() + digest.update(to_be_signed.encode('utf-8')) + sig = base64.b64encode(signer.sign(digest)) + sig = sig.decode('utf-8') + headers = { + 'Signature': 'keyId="{keyid}",algorithm="rsa-sha256",headers="{headers}",signature="{signature}"'.format( + keyid=self.keyid, + signature=sig, + headers=sigheaders, + ), + } + r.headers.update(headers) + return r diff --git a/utils/key.py b/utils/key.py new file mode 100644 index 0000000..526b3be --- /dev/null +++ b/utils/key.py @@ -0,0 +1,39 @@ +import os +import binascii + +from Crypto.PublicKey import RSA + +KEY_DIR = 'config/' + + +def get_secret_key(name:str) -> str: + key_path = f'{KEY_DIR}{name}.key' + if not os.path.exists(key_path): + k = binascii.hexlify(os.urandom(32)).decode('utf-8') + with open(key_path, 'w+') as f: + f.write(k) + return k + + with open(key_path) as f: + return f.read() + + +class Key(object): + def __init__(self, user: str, domain: str, create: bool = True) -> None: + user = user.replace('.', '_') + domain = domain.replace('.', '_') + key_path = f'{KEY_DIR}/key_{user}_{domain}.pem' + if os.path.isfile(key_path): + with open(key_path) as f: + self.privkey_pem = f.read() + self.privkey = RSA.importKey(self.privkey_pem) + self.pubkey_pem = self.privkey.publickey().exportKey('PEM').decode('utf-8') + else: + if not create: + raise Exception('must init private key first') + k = RSA.generate(4096) + self.privkey_pem = k.exportKey('PEM').decode('utf-8') + self.pubkey_pem = k.publickey().exportKey('PEM').decode('utf-8') + with open(key_path, 'w') as f: + f.write(self.privkey_pem) + self.privkey = k diff --git a/utils/linked_data_sig.py b/utils/linked_data_sig.py new file mode 100644 index 0000000..9523ed4 --- /dev/null +++ b/utils/linked_data_sig.py @@ -0,0 +1,53 @@ +from pyld import jsonld +import hashlib +from datetime import datetime + +from Crypto.Signature import PKCS1_v1_5 +from Crypto.Hash import SHA256 +import base64 + + +def options_hash(doc): + doc = dict(doc['signature']) + for k in ['type', 'id', 'signatureValue']: + if k in doc: + del doc[k] + doc['@context'] = 'https://w3id.org/identity/v1' + normalized = jsonld.normalize(doc, {'algorithm': 'URDNA2015', 'format': 'application/nquads'}) + h = hashlib.new('sha256') + h.update(normalized.encode('utf-8')) + return h.hexdigest() + + +def doc_hash(doc): + doc = dict(doc) + if 'signature' in doc: + del doc['signature'] + normalized = jsonld.normalize(doc, {'algorithm': 'URDNA2015', 'format': 'application/nquads'}) + h = hashlib.new('sha256') + h.update(normalized.encode('utf-8')) + return h.hexdigest() + + +def verify_signature(doc, pubkey): + to_be_signed = options_hash(doc) + doc_hash(doc) + signature = doc['signature']['signatureValue'] + signer = PKCS1_v1_5.new(pubkey) + digest = SHA256.new() + digest.update(to_be_signed.encode('utf-8')) + return signer.verify(digest, base64.b64decode(signature)) + + +def generate_signature(doc, privkey): + options = { + 'type': 'RsaSignature2017', + 'creator': doc['actor'] + '#main-key', + 'created': datetime.utcnow().replace(microsecond=0).isoformat() + 'Z', + } + doc['signature'] = options + to_be_signed = options_hash(doc) + doc_hash(doc) + signer = PKCS1_v1_5.new(privkey) + digest = SHA256.new() + digest.update(to_be_signed.encode('utf-8')) + sig = base64.b64encode(signer.sign(digest)) + options['signatureValue'] = sig.decode('utf-8') diff --git a/utils/object_service.py b/utils/object_service.py new file mode 100644 index 0000000..185488f --- /dev/null +++ b/utils/object_service.py @@ -0,0 +1,60 @@ +import requests +from urllib.parse import urlparse + + +class ObjectService(object): + def __init__(self, user_agent, col, inbox, outbox, instances): + self._user_agent = user_agent + self._col = col + self._inbox = inbox + self._outbox = outbox + self._instances = instances + self._known_instances = set() + + def _fetch_remote(self, object_id): + print(f'fetch remote {object_id}') + resp = requests.get(object_id, headers={ + 'Accept': 'application/activity+json', + 'User-Agent': self._user_agent, + }) + resp.raise_for_status() + return resp.json() + + def _fetch(self, object_id): + instance = urlparse(object_id)._replace(path='', query='', fragment='').geturl() + if instance not in self._known_instances: + self._known_instances.add(instance) + if not self._instances.find_one({'instance': instance}): + self._instances.insert({'instance': instance, 'first_object': object_id}) + + obj = self._inbox.find_one({'$or': [{'remote_id': object_id}, {'type': 'Create', 'activity.object.id': object_id}]}) + if obj: + if obj['remote_id'] == object_id: + return obj['activity'] + return obj['activity']['object'] + + obj = self._outbox.find_one({'$or': [{'remote_id': object_id}, {'type': 'Create', 'activity.object.id': object_id}]}) + if obj: + if obj['remote_id'] == object_id: + return obj['activity'] + return obj['activity']['object'] + + return self._fetch_remote(object_id) + + def get(self, object_id, reload_cache=False, part_of_stream=False, announce_published=None): + if reload_cache: + obj = self._fetch(object_id) + self._col.update({'object_id': object_id}, {'$set': {'cached_object': obj, 'meta.part_of_stream': part_of_stream, 'meta.announce_published': announce_published}}, upsert=True) + return obj + + cached_object = self._col.find_one({'object_id': object_id}) + if cached_object: + print(f'ObjectService: {cached_object}') + return cached_object['cached_object'] + + obj = self._fetch(object_id) + + self._col.update({'object_id': object_id}, {'$set': {'cached_object': obj, 'meta.part_of_stream': part_of_stream, 'meta.announce_published': announce_published}}, upsert=True) + # print(f'ObjectService: {obj}') + + return obj diff --git a/utils/opengraph.py b/utils/opengraph.py new file mode 100644 index 0000000..e67a5a8 --- /dev/null +++ b/utils/opengraph.py @@ -0,0 +1,46 @@ +from urllib.parse import urlparse + +import ipaddress +import opengraph +import requests +from bs4 import BeautifulSoup + +from .urlutils import is_url_valid + + +def links_from_note(note): + tags_href= set() + for t in note.get('tag', []): + h = t.get('href') + if h: + # TODO(tsileo): fetch the URL for Actor profile, type=mention + tags_href.add(h) + + links = set() + soup = BeautifulSoup(note['content']) + for link in soup.find_all('a'): + h = link.get('href') + if h.startswith('http') and h not in tags_href and is_url_valid(h): + links.add(h) + + return links + + +def fetch_og_metadata(user_agent, col, remote_id): + doc = col.find_one({'remote_id': remote_id}) + if not doc: + raise ValueError + note = doc['activity']['object'] + print(note) + links = links_from_note(note) + if not links: + return 0 + # FIXME(tsileo): set the user agent by giving HTML directly to OpenGraph + htmls = [] + for l in links: + r = requests.get(l, headers={'User-Agent': user_agent}) + r.raise_for_status() + htmls.append(r.text) + links_og_metadata = [dict(opengraph.OpenGraph(html=html)) for html in htmls] + col.update_one({'remote_id': remote_id}, {'$set': {'meta.og_metadata': links_og_metadata}}) + return len(links) diff --git a/utils/urlutils.py b/utils/urlutils.py new file mode 100644 index 0000000..b304f5e --- /dev/null +++ b/utils/urlutils.py @@ -0,0 +1,27 @@ +import logging +import socket +import ipaddress +from urllib.parse import urlparse + +logger = logging.getLogger(__name__) + + +def is_url_valid(url): + parsed = urlparse(url) + if parsed.scheme not in ['http', 'https']: + return False + + if parsed.hostname in ['localhost']: + return False + + try: + ip_address = socket.getaddrinfo(parsed.hostname, parsed.port or 80)[0][4][0] + except socket.gaierror: + logger.exception(f'failed to lookup url {url}') + return False + + if ipaddress.ip_address(ip_address).is_private: + logger.info(f'rejecting private URL {url}') + return False + + return True diff --git a/utils/webfinger.py b/utils/webfinger.py new file mode 100644 index 0000000..296ecae --- /dev/null +++ b/utils/webfinger.py @@ -0,0 +1,63 @@ +from typing import Optional +from urllib.parse import urlparse + +import requests + +def get_remote_follow_template(resource: str) -> Optional[str]: + """Mastodon-like WebFinger resolution to retrieve the activity stream Actor URL. + + Returns: + the Actor URL or None if the resolution failed. + """ + if resource.startswith('http'): + host = urlparse(resource).netloc + else: + if resource.startswith('acct:'): + resource = resource[5:] + if resource.startswith('@'): + resource = resource[1:] + _, host = resource.split('@', 1) + resource='acct:'+resource + resp = requests.get( + f'https://{host}/.well-known/webfinger', + {'resource': resource} + ) + print(resp, resp.request.url) + if resp.status_code == 404: + return None + resp.raise_for_status() + data = resp.json() + for link in data['links']: + if link.get('rel') == 'http://ostatus.org/schema/1.0/subscribe': + return link.get('template') + return None + + +def get_actor_url(resource: str) -> Optional[str]: + """Mastodon-like WebFinger resolution to retrieve the activity stream Actor URL. + + Returns: + the Actor URL or None if the resolution failed. + """ + if resource.startswith('http'): + host = urlparse(resource).netloc + else: + if resource.startswith('acct:'): + resource = resource[5:] + if resource.startswith('@'): + resource = resource[1:] + _, host = resource.split('@', 1) + resource='acct:'+resource + resp = requests.get( + f'https://{host}/.well-known/webfinger', + {'resource': resource} + ) + print(resp, resp.request.url) + if resp.status_code == 404: + return None + resp.raise_for_status() + data = resp.json() + for link in data['links']: + if link.get('rel') == 'self' and link.get('type') == 'application/activity+json': + return link.get('href') + return None From 4d6251133252e3d24026f2a2290429b8aa34c6c3 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 18 May 2018 20:44:14 +0200 Subject: [PATCH 0002/1425] Tweak the CI setup --- .travis.yml | 4 ++-- app.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e3b0785..fe27dca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,6 @@ python: install: - pip install pytest mypy flake8 script: - - flake8 - - mypy --ignore-missing-imports +# - flake8 + - mypy --ignore-missing-imports . - pytest -v diff --git a/app.py b/app.py index 034f1ca..9aece91 100644 --- a/app.py +++ b/app.py @@ -58,7 +58,7 @@ app.secret_key = get_secret_key('flask') JWT_SECRET = get_secret_key('jwt') JWT = JSONWebSignatureSerializer(JWT_SECRET) -with open('config/jwt_token', 'wb+') as f: +with open('config/jwt_token', 'w+') as f: f.write(JWT.dumps({'type': 'admin_token'})) SIG_AUTH = HTTPSigAuth(ID+'#main-key', KEY.privkey) From f6a7e5a3576f96ea7795ebacaa2475c7a2414645 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 19 May 2018 08:54:46 +0200 Subject: [PATCH 0003/1425] Fix the deploy build --- app.py | 4 ++-- docker-compose.yml | 4 ++-- static/css/theme.css | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 static/css/theme.css diff --git a/app.py b/app.py index 9aece91..70d0060 100644 --- a/app.py +++ b/app.py @@ -58,8 +58,8 @@ app.secret_key = get_secret_key('flask') JWT_SECRET = get_secret_key('jwt') JWT = JSONWebSignatureSerializer(JWT_SECRET) -with open('config/jwt_token', 'w+') as f: - f.write(JWT.dumps({'type': 'admin_token'})) +with open('config/jwt_token', 'wb+') as f: + f.write(JWT.dumps({'type': 'admin_token'})) # type: ignore SIG_AUTH = HTTPSigAuth(ID+'#main-key', KEY.privkey) diff --git a/docker-compose.yml b/docker-compose.yml index 50516aa..ad08caa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: - "./config:/app/config" - "./static:/app/static" environment: - - MICROBLOGPUB_AMQP_BORKER=pyamqp://guest@rabbitmq// + - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rabbitmq// - MICROBLOGPUB_MONGODB_HOST=mongo:27017 celery: build: . @@ -20,7 +20,7 @@ services: - rabbitmq command: 'celery worker -l info -A tasks' environment: - - MICROBLOGPUB_AMQP_BORKER=pyamqp://guest@rabbitmq// + - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rabbitmq// - MICROBLOGPUB_MONGODB_HOST=mongo:27017 mongo: image: "mongo:latest" diff --git a/static/css/theme.css b/static/css/theme.css new file mode 100644 index 0000000..4efc7a4 --- /dev/null +++ b/static/css/theme.css @@ -0,0 +1 @@ +.note-container p:first-child{margin-top:0}html,body{height:100%}body{background-color:#eee;color:#111;display:flex;flex-direction:column}.base-container{flex:1 0 auto}.footer{flex-shrink:0}a,h1,h2,h3,h4,h5,h6{color:#333}a{text-decoration:none}a:hover{text-decoration:underline}.gold{color:#1d781d}#header{margin-bottom:40px}#header .title{font-size:1.2em;padding-right:15px;color:#333}#header .title:hover{text-decoration:none}#header .subtitle-username{color:#111}#header .menu{padding:20px 0 10px 0}#header .menu ul{display:inline;list-style-type:none;padding:0}#header .menu ul li{float:left;padding-right:10px;margin-bottom:10px}#header .menu a{padding:2px 7px}#header .menu a.selected{background:#1d781d;color:#eee;border-radius:2px}#header .menu a:hover{background:#1d781d;color:#eee;text-decoration:none}#container{width:90%;max-width:720px;margin:40px auto}#container #notes{margin-top:20px}.actor-box{display:block;text-decoration:none;margin-bottom:40px}.actor-box .actor-icon{width:100%;max-width:120px;border-radius:2px}.actor-box h3{margin:0}.note{display:flex;margin-bottom:70px}.note .l{color:#333}.note .h-card{flex:initial;width:50px}.note .u-photo{width:50px;border-radius:2px}.note .note-wrapper{flex:1;padding-left:15px}.note .bottom-bar{margin-top:10px}.note .img-attachment{max-width:100%;border-radius:2px}.note h3{font-size:1.1em;color:#555}.note strong{font-weight:600}.note .note-container{clear:right;padding:10px 0}.bar-item{background:#ddd;padding:5px;color:#555;margin-right:5px;border-radius:2px}.bottom-bar .perma-item{margin-right:5px}.bottom-bar a.bar-item:hover{text-decoration:none}.footer>div{width:90%;max-width:720px;margin:40px auto}.footer a,.footer a:hover,.footer a:visited{text-decoration:underline;color:#111}.summary{color:#111;font-size:1.3em;margin-top:50px;margin-bottom:70px}.summary a,.summay a:hover{color:#111;text-decoration:underline}#followers,#following,#new{margin-top:50px}#admin{margin-top:50px}textarea,input{background:#ddd;padding:10px;color:#555;border:0px;border-radius:2px}input{padding:10px}input[type=submit]{color:#1d781d;text-transform:uppercase} From df953dd1facab0480919978cc0ea5f3018addb8d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 19 May 2018 08:59:04 +0200 Subject: [PATCH 0004/1425] Fix CI config --- .travis.yml | 2 +- templates/header.html | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index fe27dca..1ba661d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,4 +6,4 @@ install: script: # - flake8 - mypy --ignore-missing-imports . - - pytest -v +# - pytest -v diff --git a/templates/header.html b/templates/header.html index e3ca067..faa8b63 100644 --- a/templates/header.html +++ b/templates/header.html @@ -22,7 +22,8 @@
  • /admin
  • /logout
    • {% else %} -
  • /remote_follow
    • + {% endif %} From 20656259e9fbc6105e9c93b409623f49f4e9e74a Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 19 May 2018 09:10:27 +0200 Subject: [PATCH 0005/1425] Bugfix in the custom API --- app.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/app.py b/app.py index 70d0060..e7bd4c1 100644 --- a/app.py +++ b/app.py @@ -503,14 +503,24 @@ def new(): if tag['type'] == 'Mention': cc.append(tag['href']) - note = activitypub.Note( - cc=cc, - to=[to if to else config.AS_PUBLIC], - content=content, # TODO(tsileo): handle markdown - tag=tags, - source={'mediaType': 'text/markdown', 'content': source}, - inReplyTo=reply.id, - ) + if reply: + note = activitypub.Note( + cc=cc, + to=[to if to else config.AS_PUBLIC], + content=content, # TODO(tsileo): handle markdown + tag=tags, + source={'mediaType': 'text/markdown', 'content': source}, + inReplyTo=reply.id, # FIXME(tsieo): support None for inReplyTo? + ) + else: + note = activitypub.Note( + cc=cc, + to=[to if to else config.AS_PUBLIC], + content=content, # TODO(tsileo): handle markdown + tag=tags, + source={'mediaType': 'text/markdown', 'content': source}, + ) + create = note.build_create() print(create.to_dict()) create.post_to_outbox() From eb9d7c6896b6ffa27d779d2e546af6a681d075a5 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 20 May 2018 12:28:11 +0200 Subject: [PATCH 0006/1425] Tweak the README --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 071be44..33f9716 100644 --- a/README.md +++ b/README.md @@ -6,25 +6,26 @@ width="200" height="200" border="0" alt="microblog.pub">

    -Version Build Status -License +License

    A self-hosted, single-user, ActivityPub powered microblog.

    +**Still in early development.** + ## Features - Implements a basic [ActivityPub](https://activitypub.rocks/) server (with federation) - Compatible with [Mastodon](https://github.com/tootsuite/mastodon) and others (Pleroma, Hubzilla...) - Also implements a remote follow compatible with Mastodon instances - - Expose your outbox as a basic microblog - - [IndieAuth](https://indieauth.spec.indieweb.org/) endpoints (authorization and token endpoint) + - Exposes your outbox as a basic microblog + - Implements [IndieAuth](https://indieauth.spec.indieweb.org/) endpoints (authorization and token endpoint) - U2F support - You can use your ActivityPub identity to login to other websites/app - Admin UI with notifications and the stream of people you follow - - Attach files to your notes - - Privacy-aware upload that strip EXIF meta data before storing the file + - Allows you to attach files to your notes + - Privacy-aware image upload endpoint that strip EXIF meta data before storing the file - No JavaScript, that's it, even the admin UI is pure HTML/CSS - Easy to customize (the theme is written Sass) - Microformats aware (exports `h-feed`, `h-entry`, `h-cards`, ...) @@ -39,6 +40,7 @@ ```shell $ git clone $ make css +$ cp -r config/me.sample.yml config/me.yml ``` ### Configuration @@ -53,8 +55,6 @@ $ make password $ docker-compose up -d ``` -You should use a reverse proxy... - ## Development The most convenient way to hack on microblog.pub is to run the server locally, and run From 2febca5711021de3c0bb532aac276e9909c7aed8 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 20 May 2018 22:02:48 +0200 Subject: [PATCH 0007/1425] Tweak the activitypub helper --- activitypub.py | 21 +++++++++++++++++---- app.py | 25 ++++++++----------------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/activitypub.py b/activitypub.py index 2a4a937..31eab5b 100644 --- a/activitypub.py +++ b/activitypub.py @@ -162,7 +162,13 @@ class BaseActivity(object): if len(set(kwargs.keys()) - set(allowed_keys)) > 0: raise ValueError('extra data left: {}'.format(kwargs)) else: - self._data.update(**kwargs) + # Remove keys with `None` value + valid_kwargs = {} + for k, v in kwargs.items(): + if v is None: + break + valid_kwargs[k] = v + self._data.update(**valid_kwargs) def _init(self, **kwargs) -> Optional[List[str]]: raise NotImplementedError @@ -190,8 +196,15 @@ class BaseActivity(object): def type_enum(self) -> ActivityTypes: return ActivityTypes(self.type) + def _set_id(self, uri: str, obj_id: str) -> None: + raise NotImplementedError + def set_id(self, uri: str, obj_id: str) -> None: self._data['id'] = uri + try: + self._set_id(uri, obj_id) + except NotImplementedError: + pass def _actor_id(self, obj: ObjectOrIDType) -> str: if isinstance(obj, dict) and obj['type'] == ActivityTypes.PERSON.value: @@ -612,11 +625,11 @@ class Update(BaseActivity): # If the object is a Person, it means the profile was updated, we just refresh our local cache ACTOR_SERVICE.get(obj.id, reload_cache=True) - def _post_to_outbox(self, obj_id, activity, recipients): + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: obj = self.get_object() update_prefix = 'activity.object.' - update = {'$set': dict(), '$unset': dict()} + update = {'$set': dict(), '$unset': dict()} # type: Dict[str, Any] update['$set'][f'{update_prefix}updated'] = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' for k, v in obj._data.items(): if k in ['id', 'type']: @@ -638,7 +651,7 @@ class Create(BaseActivity): ACTIVITY_TYPE = ActivityTypes.CREATE ALLOWED_OBJECT_TYPES = [ActivityTypes.NOTE] - def _set_id(self, uri, obj_id): + def _set_id(self, uri: str, obj_id: str) -> None: self._data['object']['id'] = uri + '/activity' self._data['object']['url'] = ID + '/' + self.get_object().type.lower() + '/' + obj_id diff --git a/app.py b/app.py index e7bd4c1..c766a3d 100644 --- a/app.py +++ b/app.py @@ -503,23 +503,14 @@ def new(): if tag['type'] == 'Mention': cc.append(tag['href']) - if reply: - note = activitypub.Note( - cc=cc, - to=[to if to else config.AS_PUBLIC], - content=content, # TODO(tsileo): handle markdown - tag=tags, - source={'mediaType': 'text/markdown', 'content': source}, - inReplyTo=reply.id, # FIXME(tsieo): support None for inReplyTo? - ) - else: - note = activitypub.Note( - cc=cc, - to=[to if to else config.AS_PUBLIC], - content=content, # TODO(tsileo): handle markdown - tag=tags, - source={'mediaType': 'text/markdown', 'content': source}, - ) + note = activitypub.Note( + cc=cc, + to=[to if to else config.AS_PUBLIC], + content=content, # TODO(tsileo): handle markdown + tag=tags, + source={'mediaType': 'text/markdown', 'content': source}, + inReplyTo=reply.id if reply else None + ) create = note.build_create() print(create.to_dict()) From 0c8f3d085ada369324fdc46a846be19351caeacf Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 20 May 2018 22:05:50 +0200 Subject: [PATCH 0008/1425] Add docker in Travis --- .travis.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1ba661d..726fc2d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,14 @@ language: python +sudo: required python: - "3.6" +services: + - docker install: - - pip install pytest mypy flake8 + - sudo curl -L https://github.com/docker/compose/releases/download/1.21.2/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose + - sudo chmod +x /usr/local/bin/docker-compose + - docker-compose --version + - pip install pytest mypy flake8 script: # - flake8 - mypy --ignore-missing-imports . From 35007698f0525e8baf93d33393b738f9d899197b Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 20 May 2018 22:11:32 +0200 Subject: [PATCH 0009/1425] Try to spawn test instance in CI --- .travis.yml | 5 ++++- templates/index.html | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 726fc2d..7fa43fd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,9 @@ install: - docker-compose --version - pip install pytest mypy flake8 script: + - mypy --ignore-missing-imports . + - cp -r tests/me.yml config/me.yml + - docker-compose up -d + - docker-compose ps # - flake8 - - mypy --ignore-missing-imports . # - pytest -v diff --git a/templates/index.html b/templates/index.html index acf6b16..6956a99 100644 --- a/templates/index.html +++ b/templates/index.html @@ -26,6 +26,7 @@
    {% for item in outbox_data %} + {{ item }} {% if item.type == 'Announce' %} {% set boost_actor = item.activity.actor | get_actor %} From 1c1816e10270995d6bca2c873b5d623c3f32344b Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 20 May 2018 22:13:09 +0200 Subject: [PATCH 0010/1425] Add missing test files --- templates/index.html | 1 - tests/me.yml | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 tests/me.yml diff --git a/templates/index.html b/templates/index.html index 6956a99..acf6b16 100644 --- a/templates/index.html +++ b/templates/index.html @@ -26,7 +26,6 @@
    {% for item in outbox_data %} - {{ item }} {% if item.type == 'Announce' %} {% set boost_actor = item.activity.actor | get_actor %} diff --git a/tests/me.yml b/tests/me.yml new file mode 100644 index 0000000..404bf9c --- /dev/null +++ b/tests/me.yml @@ -0,0 +1,6 @@ +username: 'ci' +name: 'CI tests' +icon_url: 'https://sos-ch-dk-2.exo.io/microblogpub/microblobpub.png' +domain: 'localhost:5005' +summary: 'test instance summary' +https: false From a138bc0c489b63a323b85fd428c03d8975189c44 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 20 May 2018 22:34:56 +0200 Subject: [PATCH 0011/1425] Start the test suite --- .travis.yml | 6 +++--- tests/integration_test.py | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 tests/integration_test.py diff --git a/.travis.yml b/.travis.yml index 7fa43fd..3b33d3e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,11 +8,11 @@ install: - sudo curl -L https://github.com/docker/compose/releases/download/1.21.2/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose - docker-compose --version - - pip install pytest mypy flake8 + - pip install -r dev-requirements.txt script: - mypy --ignore-missing-imports . +# - flake8 - cp -r tests/me.yml config/me.yml - docker-compose up -d - docker-compose ps -# - flake8 -# - pytest -v + - pytest -v --ignore data diff --git a/tests/integration_test.py b/tests/integration_test.py new file mode 100644 index 0000000..3595ca4 --- /dev/null +++ b/tests/integration_test.py @@ -0,0 +1,7 @@ +import requests + +def test_ping_homepage(): + """Ensure the homepage is accessible.""" + resp = requests.get('http://localhost:5005') + resp.raise_for_status() + assert 'ci@localhost' in resp.text From 0cc936fd0b1d7b3476010696834d8869171622b6 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 20 May 2018 22:36:29 +0200 Subject: [PATCH 0012/1425] Add dev-requirements.txt --- dev-requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 dev-requirements.txt diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..9eba2d1 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,4 @@ +pytest +requests +flake8 +mypy From 388fd1cdf5633a93aa32d7e7069ec78569058954 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 20 May 2018 22:40:14 +0200 Subject: [PATCH 0013/1425] Try to debug the CI --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 3b33d3e..cfa1102 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ python: services: - docker install: + - sudo pip install -U pip - sudo curl -L https://github.com/docker/compose/releases/download/1.21.2/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose - docker-compose --version @@ -15,4 +16,6 @@ script: - cp -r tests/me.yml config/me.yml - docker-compose up -d - docker-compose ps + - sleep 2 + - docker-compose logs web - pytest -v --ignore data From c6b9bee28e9402b84cb31f015724806b447e5a92 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 20 May 2018 22:46:13 +0200 Subject: [PATCH 0014/1425] Try to debug the CI --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index cfa1102..efb28e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ python: services: - docker install: + - sudo apt-get install -y curl - sudo pip install -U pip - sudo curl -L https://github.com/docker/compose/releases/download/1.21.2/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose @@ -17,5 +18,6 @@ script: - docker-compose up -d - docker-compose ps - sleep 2 + - curl http://localhost:5005/ - docker-compose logs web - pytest -v --ignore data From 31b9010ced7ad843bbb239efd6008f8839d16d3d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 20 May 2018 22:51:06 +0200 Subject: [PATCH 0015/1425] Fix config file for the dev server --- .travis.yml | 3 +-- tests/me.yml | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index efb28e1..c83861a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,6 @@ script: - cp -r tests/me.yml config/me.yml - docker-compose up -d - docker-compose ps - - sleep 2 - - curl http://localhost:5005/ + - sleep 3 - docker-compose logs web - pytest -v --ignore data diff --git a/tests/me.yml b/tests/me.yml index 404bf9c..9aabaa3 100644 --- a/tests/me.yml +++ b/tests/me.yml @@ -3,4 +3,5 @@ name: 'CI tests' icon_url: 'https://sos-ch-dk-2.exo.io/microblogpub/microblobpub.png' domain: 'localhost:5005' summary: 'test instance summary' +pass: '$2b$12$nEgJMgaYbXSPOvgnqM4jSeYnleKhXqsFgv/o3hg12x79uEdsR4cUy' # hello https: false From 525ba4e284a251bb8fd074227a13c4a3478b78dc Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 20 May 2018 22:55:52 +0200 Subject: [PATCH 0016/1425] Fix the test suite --- tests/integration_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_test.py b/tests/integration_test.py index 3595ca4..7b8510f 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -4,4 +4,4 @@ def test_ping_homepage(): """Ensure the homepage is accessible.""" resp = requests.get('http://localhost:5005') resp.raise_for_status() - assert 'ci@localhost' in resp.text + assert resp.status_code == 200 From 3496aee23f9f911c515e957bae49de67e023c1cb Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 20 May 2018 22:56:25 +0200 Subject: [PATCH 0017/1425] Tweak the CI script --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c83861a..e41bf99 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,5 @@ script: - cp -r tests/me.yml config/me.yml - docker-compose up -d - docker-compose ps - - sleep 3 - - docker-compose logs web + - sleep 2 - pytest -v --ignore data From c9ba124bdda74245b89258b2ef9e1e9689078730 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 21 May 2018 11:21:11 +0200 Subject: [PATCH 0018/1425] Working the test suite, and fixing formatting --- .travis.yml | 2 +- activitypub.py | 71 ++++++++++++++++++++------------------- app.py | 2 +- dev-requirements.txt | 1 + tests/integration_test.py | 25 ++++++++++++-- 5 files changed, 62 insertions(+), 39 deletions(-) diff --git a/.travis.yml b/.travis.yml index e41bf99..c2bc717 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ install: - pip install -r dev-requirements.txt script: - mypy --ignore-missing-imports . -# - flake8 + - flake8 activitypub.py - cp -r tests/me.yml config/me.yml - docker-compose up -d - docker-compose ps diff --git a/activitypub.py b/activitypub.py index 31eab5b..1b51789 100644 --- a/activitypub.py +++ b/activitypub.py @@ -1,5 +1,3 @@ -import typing -import re import json import binascii import os @@ -7,16 +5,12 @@ from datetime import datetime from enum import Enum import requests -from bleach.linkifier import Linker from bson.objectid import ObjectId from html2text import html2text from feedgen.feed import FeedGenerator -from markdown import markdown from utils.linked_data_sig import generate_signature from utils.actor_service import NotAnActorError -from utils.webfinger import get_actor_url -from utils.content_helper import parse_markdown from config import USERNAME, BASE_URL, ID from config import CTX_AS, CTX_SECURITY, AS_PUBLIC from config import KEY, DB, ME, ACTOR_SERVICE @@ -24,7 +18,7 @@ from config import OBJECT_SERVICE from config import PUBLIC_INSTANCES import tasks -from typing import List, Optional, Tuple, Dict, Any, Union, Type +from typing import List, Optional, Dict, Any, Union from typing import TypeVar A = TypeVar('A', bound='BaseActivity') @@ -32,10 +26,6 @@ ObjectType = Dict[str, Any] ObjectOrIDType = Union[str, ObjectType] -# Pleroma sample -# {'@context': ['https://www.w3.org/ns/activitystreams', 'https://w3id.org/security/v1', {'Emoji': 'toot:Emoji', 'Hashtag': 'as:Hashtag', 'atomUri': 'ostatus:atomUri', 'conversation': 'ostatus:conversation', 'inReplyToAtomUri': 'ostatus:inReplyToAtomUri', 'manuallyApprovesFollowers': 'as:manuallyApprovesFollowers', 'ostatus': 'http://ostatus.org#', 'sensitive': 'as:sensitive', 'toot': 'http://joinmastodon.org/ns#'}], 'actor': 'https://soc.freedombone.net/users/bob', 'attachment': [{'mediaType': 'image/jpeg', 'name': 'stallmanlemote.jpg', 'type': 'Document', 'url': 'https://soc.freedombone.net/media/e1a3ca6f-df73-4f2d-a931-c389a221b008/stallmanlemote.jpg'}], 'attributedTo': 'https://soc.freedombone.net/users/bob', 'cc': ['https://cybre.space/users/vantablack', 'https://soc.freedombone.net/users/bob/followers'], 'content': '@vantablack
    stallmanlemote.jpg', 'context': 'tag:cybre.space,2018-04-05:objectId=5756519:objectType=Conversation', 'conversation': 'tag:cybre.space,2018-04-05:objectId=5756519:objectType=Conversation', 'emoji': {}, 'id': 'https://soc.freedombone.net/objects/3f0faeca-4d37-4acf-b990-6a50146d23cc', 'inReplyTo': 'https://cybre.space/users/vantablack/statuses/99808953472969467', 'inReplyToStatusId': 300713, 'like_count': 1, 'likes': ['https://cybre.space/users/vantablack'], 'published': '2018-04-05T21:30:52.658817Z', 'sensitive': False, 'summary': None, 'tag': [{'href': 'https://cybre.space/users/vantablack', 'name': '@vantablack@cybre.space', 'type': 'Mention'}], 'to': ['https://www.w3.org/ns/activitystreams#Public'], 'type': 'Note'} - - class ActivityTypes(Enum): ANNOUNCE = 'Announce' BLOCK = 'Block' @@ -142,7 +132,7 @@ class BaseActivity(object): if not self.NO_CONTEXT: if not isinstance(self._data['@context'], list): self._data['@context'] = [self._data['@context']] - if not CTX_SECURITY in self._data['@context']: + if CTX_SECURITY not in self._data['@context']: self._data['@context'].append(CTX_SECURITY) if isinstance(self._data['@context'][-1], dict): self._data['@context'][-1]['Hashtag'] = 'as:Hashtag' @@ -326,7 +316,6 @@ class BaseActivity(object): except NotImplementedError: pass - #return generate_signature(activity, KEY.privkey) payload = json.dumps(activity) print('will post') @@ -399,17 +388,16 @@ class Person(BaseActivity): ACTIVITY_TYPE = ActivityTypes.PERSON def _init(self, **kwargs): - #if 'icon' in kwargs: - # self._data['icon'] = Image(**kwargs.pop('icon')) + # if 'icon' in kwargs: + # self._data['icon'] = Image(**kwargs.pop('icon')) pass def _verify(self) -> None: ACTOR_SERVICE.get(self._data['id']) def _to_dict(self, data): - #if 'icon' in data: - # data['icon'] = data['icon'].to_dict() - # + # if 'icon' in data: + # data['icon'] = data['icon'].to_dict() return data @@ -512,6 +500,7 @@ class Undo(BaseActivity): except NotImplementedError: pass + class Like(BaseActivity): ACTIVITY_TYPE = ActivityTypes.LIKE ALLOWED_OBJECT_TYPES = [ActivityTypes.NOTE] @@ -567,7 +556,12 @@ class Announce(BaseActivity): return # Save/cache the object, and make it part of the stream so we can fetch it if isinstance(self._data['object'], str): - raw_obj = OBJECT_SERVICE.get(self._data['object'], reload_cache=True, part_of_stream=True, announce_published=self._data['published']) + raw_obj = OBJECT_SERVICE.get( + self._data['object'], + reload_cache=True, + part_of_stream=True, + announce_published=self._data['published'], + ) obj = parse_activity(raw_obj) else: obj = self.get_object() @@ -581,7 +575,12 @@ class Announce(BaseActivity): def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: if isinstance(self._data['object'], str): # Put the object in the cache - OBJECT_SERVICE.get(self._data['object'], reload_cache=True, part_of_stream=True, announce_published=self._data['published']) + OBJECT_SERVICE.get( + self._data['object'], + reload_cache=True, + part_of_stream=True, + announce_published=self._data['published'], + ) obj = self.get_object() DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.boosted': obj_id}}) @@ -702,7 +701,7 @@ class Create(BaseActivity): parent = DB.inbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) if parent is None: # The reply is a note from the outbox - data = DB.outbox.update_one( + DB.outbox.update_one( {'activity.object.id': in_reply_to}, {'$inc': {'meta.count_reply': 1}}, ) @@ -732,7 +731,7 @@ class Note(BaseActivity): # for t in kwargs.get('tag', []): # if t['type'] == 'Mention': # cc -> c['href'] - + def _recipients(self) -> List[str]: # TODO(tsileo): audience support? recipients = [] # type: List[str] @@ -772,6 +771,7 @@ class Note(BaseActivity): published=datetime.utcnow().replace(microsecond=0).isoformat() + 'Z', ) + _ACTIVITY_TYPE_TO_CLS = { ActivityTypes.IMAGE: Image, ActivityTypes.PERSON: Person, @@ -789,6 +789,7 @@ _ACTIVITY_TYPE_TO_CLS = { ActivityTypes.TOMBSTONE: Tombstone, } + def parse_activity(payload: ObjectType) -> BaseActivity: t = ActivityTypes(payload['type']) if t not in _ACTIVITY_TYPE_TO_CLS: @@ -801,11 +802,10 @@ def gen_feed(): fg = FeedGenerator() fg.id(f'{ID}') fg.title(f'{USERNAME} notes') - fg.author( {'name': USERNAME,'email':'t@a4.io'} ) + fg.author({'name': USERNAME, 'email': 't@a4.io'}) fg.link(href=ID, rel='alternate') fg.description(f'{USERNAME} notes') fg.logo(ME.get('icon', {}).get('url')) - #fg.link( href='http://larskiesow.de/test.atom', rel='self' ) fg.language('en') for item in DB.outbox.find({'type': 'Create'}, limit=50): fe = fg.add_entry() @@ -829,7 +829,8 @@ def json_feed(path: str) -> Dict[str, Any]: }) return { "version": "https://jsonfeed.org/version/1", - "user_comment": "This is a microblog feed. You can add this to your feed reader using the following URL: " + ID + path, + "user_comment": ("This is a microblog feed. You can add this to your feed reader using the following URL: " + + ID + path), "title": USERNAME, "home_page_url": ID, "feed_url": ID + path, @@ -958,17 +959,17 @@ def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, # No cursor, this is the first page and we return an OrderedCollection if not cursor: resp = { - '@context': CTX_AS, - 'first': { - 'id': BASE_URL + '/' + col_name + '?cursor=' + start_cursor, - 'orderedItems': data, - 'partOf': BASE_URL + '/' + col_name, - 'totalItems': total_items, - 'type': 'OrderedCollectionPage' - }, - 'id': BASE_URL + '/' + col_name, + '@context': CTX_AS, + 'id': f'{BASE_URL}/{col_name}', 'totalItems': total_items, - 'type': 'OrderedCollection' + 'type': 'OrderedCollection', + 'first': { + 'id': f'{BASE_URL}/{col_name}?cursor={start_cursor}', + 'orderedItems': data, + 'partOf': f'{BASE_URL}/{col_name}', + 'totalItems': total_items, + 'type': 'OrderedCollectionPage' + }, } if len(data) == limit: diff --git a/app.py b/app.py index c766a3d..55e2d47 100644 --- a/app.py +++ b/app.py @@ -34,7 +34,7 @@ import activitypub import config from activitypub import ActivityTypes from activitypub import clean_activity -from activitypub import parse_markdown +from utils.content_helper import parse_markdown from config import KEY from config import DB from config import ME diff --git a/dev-requirements.txt b/dev-requirements.txt index 9eba2d1..dc80f66 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,4 +1,5 @@ pytest requests +html2text flake8 mypy diff --git a/tests/integration_test.py b/tests/integration_test.py index 7b8510f..4270b4b 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -1,7 +1,28 @@ -import requests +import os -def test_ping_homepage(): +import pytest +import requests +from html2text import html2text + + +@pytest.fixture +def config(): + """Return the current config as a dict.""" + import yaml + with open(os.path.join(os.path.dirname(__file__), '..', 'config/me.yml'), 'rb') as f: + yield yaml.load(f) + + +def resp2plaintext(resp): + """Convert the body of a requests reponse to plain text in order to make basic assertions.""" + return html2text(resp.text) + + +def test_ping_homepage(config): """Ensure the homepage is accessible.""" resp = requests.get('http://localhost:5005') resp.raise_for_status() assert resp.status_code == 200 + body = resp2plaintext(resp) + assert config['name'] in body + assert f"@{config['username']}@{config['domain']}" in body From eaf947fc3c9e7ffc0c73572a86e64f1c4ce16f1b Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 21 May 2018 11:53:58 +0200 Subject: [PATCH 0019/1425] Tweak docker-compose to allow starting multiple instances --- .env | 3 +++ docker-compose.yml | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 0000000..70fb2a3 --- /dev/null +++ b/.env @@ -0,0 +1,3 @@ +WEB_PORT=5005 +CONFIG_DIR=./config +DATA_DIR=./data diff --git a/docker-compose.yml b/docker-compose.yml index ad08caa..5ce5dee 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,12 +3,12 @@ services: web: build: . ports: - - "5005:5005" + - "${WEB_PORT}:5005" links: - mongo - rabbitmq volumes: - - "./config:/app/config" + - "${CONFIG_DIR}:/app/config" - "./static:/app/static" environment: - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rabbitmq// @@ -25,6 +25,6 @@ services: mongo: image: "mongo:latest" volumes: - - "./data:/data/db" + - "${DATA_DIR}:/data/db" rabbitmq: image: "rabbitmq:latest" From fc1219860b89b3dfa386591efd37ade427d42762 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 21 May 2018 11:57:25 +0200 Subject: [PATCH 0020/1425] Fix dev requirements --- dev-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index dc80f66..62e71f2 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,6 @@ pytest requests html2text +pyyaml flake8 mypy From 77a3197ad647e47005b3306fd19cfd14ecc6f2d7 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 21 May 2018 12:18:15 +0200 Subject: [PATCH 0021/1425] Prepare for federation testing in the CI --- .travis.yml | 7 +++++-- Dockerfile | 2 +- tests/fixtures/instance1/config/.gitignore | 2 ++ tests/fixtures/instance1/data/.gitignore | 2 ++ tests/fixtures/instance2/config/.gitignore | 2 ++ tests/fixtures/instance2/data/.gitignore | 2 ++ tests/{ => fixtures}/me.yml | 0 7 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/instance1/config/.gitignore create mode 100644 tests/fixtures/instance1/data/.gitignore create mode 100644 tests/fixtures/instance2/config/.gitignore create mode 100644 tests/fixtures/instance2/data/.gitignore rename tests/{ => fixtures}/me.yml (100%) diff --git a/.travis.yml b/.travis.yml index c2bc717..ea656ce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,8 +14,11 @@ install: script: - mypy --ignore-missing-imports . - flake8 activitypub.py - - cp -r tests/me.yml config/me.yml + - cp -r tests/fixtures/me.yml config/me.yml - docker-compose up -d - docker-compose ps - - sleep 2 + - WEB_PORT=5006 DATA_DIR=./tests/fixtures/instance1/data CONFIG_DIR=./tests/fixtures/instance1/config docker-compose -p instance1 up + - docker-compose -p instance1 ps + - WEB_PORT=5007 DATA_DIR=./tests/fixtures/instance2/data CONFIG_DIR=./tests/fixtures/instance2/config docker-compose -p instance2 up + - docker-compose -p instance2 ps - pytest -v --ignore data diff --git a/Dockerfile b/Dockerfile index 3fed815..ae59305 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,4 +3,4 @@ ADD . /app WORKDIR /app RUN pip install -r requirements.txt ENV FLASK_APP=app.py -CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5005", "app:app"] +CMD ["gunicorn", "-w", "2", "-b", "0.0.0.0:5005", "app:app"] diff --git a/tests/fixtures/instance1/config/.gitignore b/tests/fixtures/instance1/config/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/tests/fixtures/instance1/config/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/tests/fixtures/instance1/data/.gitignore b/tests/fixtures/instance1/data/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/tests/fixtures/instance1/data/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/tests/fixtures/instance2/config/.gitignore b/tests/fixtures/instance2/config/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/tests/fixtures/instance2/config/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/tests/fixtures/instance2/data/.gitignore b/tests/fixtures/instance2/data/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/tests/fixtures/instance2/data/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/tests/me.yml b/tests/fixtures/me.yml similarity index 100% rename from tests/me.yml rename to tests/fixtures/me.yml From 4a0a35140297bb8c3b05fc6550c079aee6b12ada Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 21 May 2018 12:23:53 +0200 Subject: [PATCH 0022/1425] Fix the CI --- .travis.yml | 4 ++-- tests/fixtures/instance1/config/me.yml | 7 +++++++ tests/fixtures/instance2/config/me.yml | 7 +++++++ 3 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/instance1/config/me.yml create mode 100644 tests/fixtures/instance2/config/me.yml diff --git a/.travis.yml b/.travis.yml index ea656ce..37e88fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,8 +17,8 @@ script: - cp -r tests/fixtures/me.yml config/me.yml - docker-compose up -d - docker-compose ps - - WEB_PORT=5006 DATA_DIR=./tests/fixtures/instance1/data CONFIG_DIR=./tests/fixtures/instance1/config docker-compose -p instance1 up + - WEB_PORT=5006 DATA_DIR=tests/fixtures/instance1/data CONFIG_DIR=tests/fixtures/instance1/config docker-compose -p instance1 up -d - docker-compose -p instance1 ps - - WEB_PORT=5007 DATA_DIR=./tests/fixtures/instance2/data CONFIG_DIR=./tests/fixtures/instance2/config docker-compose -p instance2 up + - WEB_PORT=5007 DATA_DIR=tests/fixtures/instance2/data CONFIG_DIR=tests/fixtures/instance2/config docker-compose -p instance2 up -d - docker-compose -p instance2 ps - pytest -v --ignore data diff --git a/tests/fixtures/instance1/config/me.yml b/tests/fixtures/instance1/config/me.yml new file mode 100644 index 0000000..2513938 --- /dev/null +++ b/tests/fixtures/instance1/config/me.yml @@ -0,0 +1,7 @@ +username: 'instance_1' +name: 'Instance 1' +icon_url: 'https://sos-ch-dk-2.exo.io/microblogpub/microblobpub.png' +domain: 'localhost:5006' +summary: 'instance 1 summary' +pass: '$2b$12$nEgJMgaYbXSPOvgnqM4jSeYnleKhXqsFgv/o3hg12x79uEdsR4cUy' # hello +https: false diff --git a/tests/fixtures/instance2/config/me.yml b/tests/fixtures/instance2/config/me.yml new file mode 100644 index 0000000..844e5c8 --- /dev/null +++ b/tests/fixtures/instance2/config/me.yml @@ -0,0 +1,7 @@ +username: 'instance_2' +name: 'Instance 2' +icon_url: 'https://sos-ch-dk-2.exo.io/microblogpub/microblobpub.png' +domain: 'localhost:5007' +summary: 'instance 2 summary' +pass: '$2b$12$nEgJMgaYbXSPOvgnqM4jSeYnleKhXqsFgv/o3hg12x79uEdsR4cUy' # hello +https: false From 6e485fbad432949a601d219490df85c891842260 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 21 May 2018 12:30:55 +0200 Subject: [PATCH 0023/1425] Tweak the Dockerfile --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index 8fce603..35249de 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,2 @@ data/ +tests/ From d3f63979991e10ba393caf6e80e3da793b703fcb Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 21 May 2018 13:00:17 +0200 Subject: [PATCH 0024/1425] Add persistency to RabbitMQ via Docker --- .../instance1/data => data/mongodb}/.gitignore | 0 .../data => data/rabbitmq}/.gitignore | 0 docker-compose.yml | 18 ++++++++++++------ 3 files changed, 12 insertions(+), 6 deletions(-) rename {tests/fixtures/instance1/data => data/mongodb}/.gitignore (100%) rename {tests/fixtures/instance2/data => data/rabbitmq}/.gitignore (100%) diff --git a/tests/fixtures/instance1/data/.gitignore b/data/mongodb/.gitignore similarity index 100% rename from tests/fixtures/instance1/data/.gitignore rename to data/mongodb/.gitignore diff --git a/tests/fixtures/instance2/data/.gitignore b/data/rabbitmq/.gitignore similarity index 100% rename from tests/fixtures/instance2/data/.gitignore rename to data/rabbitmq/.gitignore diff --git a/docker-compose.yml b/docker-compose.yml index 5ce5dee..c2dab44 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,25 +6,31 @@ services: - "${WEB_PORT}:5005" links: - mongo - - rabbitmq + - rmq volumes: - "${CONFIG_DIR}:/app/config" - "./static:/app/static" environment: - - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rabbitmq// + - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// - MICROBLOGPUB_MONGODB_HOST=mongo:27017 celery: build: . links: - mongo - - rabbitmq + - rmq command: 'celery worker -l info -A tasks' environment: - - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rabbitmq// + - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// - MICROBLOGPUB_MONGODB_HOST=mongo:27017 mongo: image: "mongo:latest" volumes: - - "${DATA_DIR}:/data/db" - rabbitmq: + - "${DATA_DIR}/mongodb:/data/db" + rmq: image: "rabbitmq:latest" + hostname: "my-rabbit" + environment: + - RABBITMQ_ERLANG_COOKIE=secretrabbit + - RABBITMQ_NODENAME=rabbit@my-rabbit + volumes: + - "${DATA_DIR}/rabbitmq:/var/lib/rabbitmq" From 95f6b5e2145b90b0db46fa4676c2a3f91f2266da Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 21 May 2018 13:03:44 +0200 Subject: [PATCH 0025/1425] No data volumes for tests instances in Docker --- .travis.yml | 8 ++++---- docker-compose-tests.yml | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 docker-compose-tests.yml diff --git a/.travis.yml b/.travis.yml index 37e88fa..af5b21a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,8 +17,8 @@ script: - cp -r tests/fixtures/me.yml config/me.yml - docker-compose up -d - docker-compose ps - - WEB_PORT=5006 DATA_DIR=tests/fixtures/instance1/data CONFIG_DIR=tests/fixtures/instance1/config docker-compose -p instance1 up -d - - docker-compose -p instance1 ps - - WEB_PORT=5007 DATA_DIR=tests/fixtures/instance2/data CONFIG_DIR=tests/fixtures/instance2/config docker-compose -p instance2 up -d - - docker-compose -p instance2 ps + - WEB_PORT=5006 CONFIG_DIR=./tests/fixtures/instance1/config docker-compose -p instance1 -f docker-compose-tests.yml up -d + - docker-compose -p instance1 -f docker-compose-tests.yml ps + - WEB_PORT=5007 CONFIG_DIR=./tests/fixtures/instance2/config docker-compose -p instance2 -f docker-compose-tests.yml up -d + - docker-compose -p instance2 -f docker-compose-tests.yml ps - pytest -v --ignore data diff --git a/docker-compose-tests.yml b/docker-compose-tests.yml new file mode 100644 index 0000000..d41baf8 --- /dev/null +++ b/docker-compose-tests.yml @@ -0,0 +1,32 @@ +version: '3' +services: + web: + build: . + ports: + - "${WEB_PORT}:5005" + links: + - mongo + - rmq + volumes: + - "${CONFIG_DIR}:/app/config" + - "./static:/app/static" + environment: + - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// + - MICROBLOGPUB_MONGODB_HOST=mongo:27017 + celery: + build: . + links: + - mongo + - rmq + command: 'celery worker -l info -A tasks' + environment: + - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// + - MICROBLOGPUB_MONGODB_HOST=mongo:27017 + mongo: + image: "mongo:latest" + rmq: + image: "rabbitmq:latest" + hostname: "my-rabbit" + environment: + - RABBITMQ_ERLANG_COOKIE=secretrabbit + - RABBITMQ_NODENAME=rabbit@my-rabbit From dc9df98084a108ea007c4ea1562e23f393b354aa Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 21 May 2018 14:30:52 +0200 Subject: [PATCH 0026/1425] Fix auth in some endpoint --- app.py | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/app.py b/app.py index 55e2d47..8895019 100644 --- a/app.py +++ b/app.py @@ -178,24 +178,26 @@ def login_required(f): return f(*args, **kwargs) return decorated_function +def _api_required(): + if session.get('logged_in'): + return -def api_required(f): + # Token verification + token = request.headers.get('Authorization', '').replace('Bearer ', '') + if not token: + token = request.form.get('access_token', '') + + # Will raise a BadSignature on bad auth + payload = JWT.loads(token) + def api_required(f): @wraps(f) def decorated_function(*args, **kwargs): - if session.get('logged_in'): - return f(*args, **kwargs) - - # Token verification - token = request.headers.get('Authorization', '').replace('Bearer ', '') - if not token: - token = request.form.get('access_token', '') - try: - payload = JWT.loads(token) - # TODO(tsileo): log payload + _api_required() except BadSignature: abort(401) - return f(*args, **kwargs) + + return f(*args, **kwargs) return decorated_function @@ -434,7 +436,11 @@ def outbox(): )) # Handle POST request - # FIXME(tsileo): check auth + try: + _api_required() + except BadSignature: + abort(401) + data = request.get_json(force=True) print(data) activity = activitypub.parse_activity(data) @@ -653,14 +659,18 @@ def inbox(): if request.method == 'GET': if not is_api_request(): abort(404) - # TODO(tsileo): handle auth and only return 404 if unauthenticated - # abort(404) + try: + _api_required() + except BadSignature: + abort(404) + return jsonify(**activitypub.build_ordered_collection( DB.inbox, q={'meta.deleted': False}, cursor=request.args.get('cursor'), map_func=lambda doc: doc['activity'], )) + data = request.get_json(force=True) # FIXME(tsileo): ensure verify_request() == True print(data) From 67643482c888e69da2c46860aa7e2a665400d4bb Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 21 May 2018 14:41:47 +0200 Subject: [PATCH 0027/1425] Improve the request verification checking --- app.py | 15 +++++++++++---- utils/httpsig.py | 6 +----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/app.py b/app.py index 8895019..7763b01 100644 --- a/app.py +++ b/app.py @@ -178,6 +178,7 @@ def login_required(f): return f(*args, **kwargs) return decorated_function + def _api_required(): if session.get('logged_in'): return @@ -189,7 +190,9 @@ def _api_required(): # Will raise a BadSignature on bad auth payload = JWT.loads(token) - def api_required(f): + + +def api_required(f): @wraps(f) def decorated_function(*args, **kwargs): try: @@ -197,7 +200,7 @@ def _api_required(): except BadSignature: abort(401) - return f(*args, **kwargs) + return f(*args, **kwargs) return decorated_function @@ -672,12 +675,16 @@ def inbox(): )) data = request.get_json(force=True) - # FIXME(tsileo): ensure verify_request() == True print(data) try: print(verify_request(ACTOR_SERVICE)) except Exception: - print('failed to verify request') + print('failed to verify request, trying to verify the payload by fetching the remote') + try: + data = OBJECT_SERVICE.get(data['id']) + except Exception: + print(f'failed to fetch remote id at {data["id"]}') + abort(422) activity = activitypub.parse_activity(data) print(activity) diff --git a/utils/httpsig.py b/utils/httpsig.py index a2e77c5..84245fb 100644 --- a/utils/httpsig.py +++ b/utils/httpsig.py @@ -77,11 +77,7 @@ class HTTPSigAuth(AuthBase): sig = base64.b64encode(signer.sign(digest)) sig = sig.decode('utf-8') headers = { - 'Signature': 'keyId="{keyid}",algorithm="rsa-sha256",headers="{headers}",signature="{signature}"'.format( - keyid=self.keyid, - signature=sig, - headers=sigheaders, - ), + 'Signature': f'keyId="{self.keyid}",algorithm="rsa-sha256",headers="{sigheaders}",signature="{sig}"' } r.headers.update(headers) return r From 56ad72148b5feed559b55c46d4c69cfbf2f325d1 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 21 May 2018 17:04:53 +0200 Subject: [PATCH 0028/1425] Cleanup, add helper for external caching --- activitypub.py | 32 ++++++++++++++++++++++---------- app.py | 11 ++++++++++- config.py | 12 +++++++++++- templates/layout.html | 2 +- 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/activitypub.py b/activitypub.py index 1b51789..e00a18f 100644 --- a/activitypub.py +++ b/activitypub.py @@ -81,9 +81,8 @@ def _get_actor_id(actor: ObjectOrIDType) -> str: class BaseActivity(object): - ACTIVITY_TYPE = None # type: Optional[ActivityTypes] - NO_CONTEXT = False - ALLOWED_OBJECT_TYPES = None # type: List[ActivityTypes] + ACTIVITY_TYPE: Optional[ActivityTypes] = None + ALLOWED_OBJECT_TYPES: List[ActivityTypes] = [] def __init__(self, **kwargs) -> None: if not self.ACTIVITY_TYPE: @@ -92,7 +91,7 @@ class BaseActivity(object): if kwargs.get('type') is not None and kwargs.pop('type') != self.ACTIVITY_TYPE.value: raise ValueError('Expect the type to be {}'.format(self.ACTIVITY_TYPE)) - self._data = {'type': self.ACTIVITY_TYPE.value} # type: Dict[str, Any] + self._data: Dict[str, Any] = {'type': self.ACTIVITY_TYPE.value} if 'id' in kwargs: self._data['id'] = kwargs.pop('id') @@ -230,7 +229,7 @@ class BaseActivity(object): p = parse_activity(obj) - self.__obj = p # type: BaseActivity + self.__obj: BaseActivity = p return p def _to_dict(self, data: ObjectType) -> ObjectType: @@ -267,6 +266,9 @@ class BaseActivity(object): def _undo_inbox(self) -> None: raise NotImplementedError + def _should_purge_cache(self) -> bool: + raise NotImplementedError + def process_from_inbox(self) -> None: self.verify() actor = self.get_actor() @@ -332,7 +334,7 @@ class BaseActivity(object): def recipients(self) -> List[str]: recipients = self._recipients() - out = [] # type: List[str] + out: List[str] = [] for recipient in recipients: if recipient in PUBLIC_INSTANCES: if recipient not in out: @@ -455,6 +457,10 @@ class Follow(BaseActivity): def build_undo(self) -> BaseActivity: return Undo(object=self.to_dict(embed=True)) + def _should_purge_cache(self) -> bool: + # Receiving a follow activity in the inbox should reset the application cache + return True + class Accept(BaseActivity): ACTIVITY_TYPE = ActivityTypes.ACCEPT @@ -468,6 +474,12 @@ class Accept(BaseActivity): if DB.following.find({'remote_actor': remote_actor}).count() == 0: DB.following.insert_one({'remote_actor': remote_actor}) + def _should_purge_cache(self) -> bool: + # Receiving an accept activity in the inbox should reset the application cache + # (a follow request has been accepted) + return True + + class Undo(BaseActivity): ACTIVITY_TYPE = ActivityTypes.UNDO @@ -628,7 +640,7 @@ class Update(BaseActivity): obj = self.get_object() update_prefix = 'activity.object.' - update = {'$set': dict(), '$unset': dict()} # type: Dict[str, Any] + update: Dict[str, Any] = {'$set': dict(), '$unset': dict()} update['$set'][f'{update_prefix}updated'] = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' for k, v in obj._data.items(): if k in ['id', 'type']: @@ -734,7 +746,7 @@ class Note(BaseActivity): def _recipients(self) -> List[str]: # TODO(tsileo): audience support? - recipients = [] # type: List[str] + recipients: List[str] = [] # If the note is public, we publish it to the defined "public instances" if AS_PUBLIC in self._data.get('to', []): @@ -847,7 +859,7 @@ def build_inbox_json_feed(path: str, request_cursor: Optional[str] = None) -> Di data = [] cursor = None - q = {'type': 'Create'} # type: Dict[str, Any] + q: Dict[str, Any] = {'type': 'Create'} if request_cursor: q['_id'] = {'$lt': request_cursor} @@ -889,7 +901,7 @@ def parse_collection(payload: Optional[Dict[str, Any]] = None, url: Optional[str return [doc['remote_actor'] for doc in DB.following.find()] # Go through all the pages - out = [] # type: List[str] + out: List[str] = [] if url: resp = requests.get(url, headers={'Accept': 'application/activity+json'}) resp.raise_for_status() diff --git a/app.py b/app.py index 7763b01..e521e2a 100644 --- a/app.py +++ b/app.py @@ -46,6 +46,8 @@ from config import ACTOR_SERVICE from config import OBJECT_SERVICE from config import PASS from config import HEADERS +from config import VERSION +from config import custom_cache_purge_hook from utils.httpsig import HTTPSigAuth, verify_request from utils.key import get_secret_key from utils.webfinger import get_remote_follow_template @@ -69,7 +71,11 @@ def verify_pass(pwd): @app.context_processor def inject_config(): - return dict(config=config, logged_in=session.get('logged_in', False)) + return dict( + microblogpub_version=VERSION, + config=config, + logged_in=session.get('logged_in', False), + ) @app.after_request def set_x_powered_by(response): @@ -453,6 +459,9 @@ def outbox(): activity.post_to_outbox() + # Purge the cache if a custom hook is set, as new content was published + custom_cache_purge_hook() + return Response(status=201, headers={'Location': activity.id}) diff --git a/config.py b/config.py index b8ff84f..250916c 100644 --- a/config.py +++ b/config.py @@ -1,3 +1,4 @@ +import subprocess import os import yaml from pymongo import MongoClient @@ -7,8 +8,17 @@ from utils.key import Key from utils.actor_service import ActorService from utils.object_service import ObjectService +def noop(): + pass -VERSION = '1.0.0' + +CUSTOM_CACHE_HOOKS = False +try: + from cache_hooks import purge as custom_cache_purge_hook +except ModuleNotFoundError: + custom_cache_purge_hook = noop + +VERSION = subprocess.check_output(['git', 'describe', '--always']).split()[0].decode('utf-8') CTX_AS = 'https://www.w3.org/ns/activitystreams' CTX_SECURITY = 'https://w3id.org/security/v1' diff --git a/templates/layout.html b/templates/layout.html index a50d6fb..25f3880 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -22,7 +22,7 @@
    - Powered by microblog.pub (source code) and the ActivityPub protocol + Powered by microblog.pub {{ microblogpub_version }} (source code) and the ActivityPub protocol
    From 4d9fd0fd03f379369d24fe088435eb1adf33be0a Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 21 May 2018 17:21:32 +0200 Subject: [PATCH 0029/1425] Finish the basic purge external cache support --- activitypub.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/activitypub.py b/activitypub.py index e00a18f..9e07401 100644 --- a/activitypub.py +++ b/activitypub.py @@ -266,6 +266,9 @@ class BaseActivity(object): def _undo_inbox(self) -> None: raise NotImplementedError + def _undo_should_purge_cache(self) -> bool: + raise NotImplementedError + def _should_purge_cache(self) -> bool: raise NotImplementedError @@ -503,6 +506,16 @@ class Undo(BaseActivity): except NotImplementedError: pass + def _should_purge_cache(self) -> bool: + obj = self.get_object() + try: + # Receiving a undo activity regarding an activity that was mentioning a published activity should purge the cache + return obj._undo_should_purge_cache() + except NotImplementedError: + pass + + return False + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: obj = self.get_object() DB.outbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) @@ -530,6 +543,10 @@ class Like(BaseActivity): # Update the meta counter if the object is published by the server DB.outbox.update_one({'activity.object.id': obj.id}, {'$inc': {'meta.count_like': -1}}) + def _undo_should_purge_cache(self) -> bool: + # If a like coutn was decremented, we need to purge the application cache + return self.get_object().id.startswith(BASE_URL) + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]): obj = self.get_object() # Unlikely, but an actor can like it's own post @@ -584,6 +601,10 @@ class Announce(BaseActivity): DB.inbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) DB.outbox.update_one({'activity.object.id': obj.id}, {'$inc': {'meta.count_boost': -1}}) + def _undo_should_purge_cache(self) -> bool: + # If a like coutn was decremented, we need to purge the application cache + return self.get_object().id.startswith(BASE_URL) + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: if isinstance(self._data['object'], str): # Put the object in the cache @@ -615,6 +636,7 @@ class Delete(BaseActivity): def _process_from_inbox(self): DB.inbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) # TODO(tsileo): also delete copies stored in parents' `meta.replies` + # TODO(tsileo): also purge the cache if it's a reply of a published activity def _post_to_outbox(self, obj_id, activity, recipients): DB.outbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) @@ -636,6 +658,8 @@ class Update(BaseActivity): # If the object is a Person, it means the profile was updated, we just refresh our local cache ACTOR_SERVICE.get(obj.id, reload_cache=True) + # TODO(tsileo): implements _should_purge_cache if it's a reply of a published activity (i.e. in the outbox) + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: obj = self.get_object() @@ -720,6 +744,17 @@ class Create(BaseActivity): else: parent = None + def _should_purge_cache(self) -> bool: + # TODO(tsileo): handle reply of a reply... + obj = self.get_object() + in_reply_to = obj.inReplyTo + if in_reply_to: + local_activity = DB.outbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) + if local_activity: + return True + + return False + class Tombstone(BaseActivity): ACTIVITY_TYPE = ActivityTypes.TOMBSTONE From a7e408028a1c38d6224916babc4a70fccfe94b7e Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 21 May 2018 17:23:54 +0200 Subject: [PATCH 0030/1425] Update the README --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 33f9716..30b64c8 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,9 @@ - Microformats aware (exports `h-feed`, `h-entry`, `h-cards`, ...) - Exports RSS/Atom feeds - Comes with a tiny HTTP API to help posting new content and performing basic actions + - Easy to "cache" (the external/public-facing microblog part) + - With a good setup, cached content should returned most of the time + - You can setup a "purge" hook to let you invalidate cache when the microblog was updated - Deployable with Docker ## Running your instance From f0264241ee0b691c0547c068fb2a5a5dd4a87347 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 21 May 2018 17:24:38 +0200 Subject: [PATCH 0031/1425] Fix the README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 30b64c8..c93a814 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ - Exports RSS/Atom feeds - Comes with a tiny HTTP API to help posting new content and performing basic actions - Easy to "cache" (the external/public-facing microblog part) - - With a good setup, cached content should returned most of the time + - With a good setup, cached content can be served most of the time - You can setup a "purge" hook to let you invalidate cache when the microblog was updated - Deployable with Docker From 08077b58f0aab393599ee72c7f075b51ab96b9ff Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 21 May 2018 17:26:48 +0200 Subject: [PATCH 0032/1425] Fix flake8 warning --- activitypub.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/activitypub.py b/activitypub.py index 9e07401..227f5be 100644 --- a/activitypub.py +++ b/activitypub.py @@ -337,7 +337,7 @@ class BaseActivity(object): def recipients(self) -> List[str]: recipients = self._recipients() - out: List[str] = [] + out: List[str] = [] for recipient in recipients: if recipient in PUBLIC_INSTANCES: if recipient not in out: @@ -483,7 +483,6 @@ class Accept(BaseActivity): return True - class Undo(BaseActivity): ACTIVITY_TYPE = ActivityTypes.UNDO ALLOWED_OBJECT_TYPES = [ActivityTypes.FOLLOW, ActivityTypes.LIKE, ActivityTypes.ANNOUNCE] @@ -509,7 +508,8 @@ class Undo(BaseActivity): def _should_purge_cache(self) -> bool: obj = self.get_object() try: - # Receiving a undo activity regarding an activity that was mentioning a published activity should purge the cache + # Receiving a undo activity regarding an activity that was mentioning a published activity + # should purge the cache return obj._undo_should_purge_cache() except NotImplementedError: pass @@ -754,7 +754,7 @@ class Create(BaseActivity): return True return False - + class Tombstone(BaseActivity): ACTIVITY_TYPE = ActivityTypes.TOMBSTONE From 51690952407eafe517c504f351dc89f986b9f040 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 21 May 2018 17:29:31 +0200 Subject: [PATCH 0033/1425] Fix formatting --- activitypub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitypub.py b/activitypub.py index 227f5be..5cf1bee 100644 --- a/activitypub.py +++ b/activitypub.py @@ -754,7 +754,7 @@ class Create(BaseActivity): return True return False - + class Tombstone(BaseActivity): ACTIVITY_TYPE = ActivityTypes.TOMBSTONE From abc168257ef64350c94fb13a29daab017b27e8ec Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 21 May 2018 21:03:21 +0200 Subject: [PATCH 0034/1425] Better title on the website --- templates/admin.html | 1 + templates/followers.html | 1 + templates/following.html | 1 + templates/layout.html | 2 +- templates/login.html | 1 + templates/new.html | 1 + templates/note.html | 3 ++- templates/stream.html | 1 + templates/tags.html | 1 + 9 files changed, 10 insertions(+), 2 deletions(-) diff --git a/templates/admin.html b/templates/admin.html index 90e11c7..e573e02 100644 --- a/templates/admin.html +++ b/templates/admin.html @@ -1,5 +1,6 @@ {% extends "layout.html" %} {% import 'utils.html' as utils %} +{% block title %}Admin - {{ config.NAME }}{% endblock %} {% block content %}
    {% include "header.html" %} diff --git a/templates/followers.html b/templates/followers.html index c4d03e2..e4f43a1 100644 --- a/templates/followers.html +++ b/templates/followers.html @@ -1,5 +1,6 @@ {% extends "layout.html" %} {% import 'utils.html' as utils %} +{% block title %}Followers - {{ config.NAME }}{% endblock %} {% block header %} {% endblock %} {% block content %} diff --git a/templates/following.html b/templates/following.html index c783133..83e8b0d 100644 --- a/templates/following.html +++ b/templates/following.html @@ -1,5 +1,6 @@ {% extends "layout.html" %} {% import 'utils.html' as utils %} +{% block title %}Following - {{ config.NAME }}{% endblock %} {% block header %} {% endblock %} {% block content %} diff --git a/templates/layout.html b/templates/layout.html index 25f3880..648c06b 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -4,7 +4,7 @@ -{{ config.USERNAME }} +{% block title %}{{ config.NAME }}{% endblock %} - microblog.pub diff --git a/templates/login.html b/templates/login.html index 2c16998..7e3cc1c 100644 --- a/templates/login.html +++ b/templates/login.html @@ -1,5 +1,6 @@ {% extends "layout.html" %} {% import 'utils.html' as utils %} +{% block title %}Login - {{ config.NAME }}{% endblock %} {% block header %} {% endblock %} {% block content %} diff --git a/templates/new.html b/templates/new.html index 2e73f38..572eb61 100644 --- a/templates/new.html +++ b/templates/new.html @@ -1,5 +1,6 @@ {% extends "layout.html" %} {% import 'utils.html' as utils %} +{% block title %}New - {{ config.NAME }}{% endblock %} {% block content %}
    {% include "header.html" %} diff --git a/templates/note.html b/templates/note.html index 55bdfa2..9311c43 100644 --- a/templates/note.html +++ b/templates/note.html @@ -1,12 +1,13 @@ {% extends "layout.html" %} {% import 'utils.html' as utils %} +{% block title %}{{ config.NAME }}: "{{ note.activity.object.content | html2plaintext | truncate(50) }}"{% endblock %} {% block header %} - + diff --git a/templates/stream.html b/templates/stream.html index 4759ae0..9a0cff7 100644 --- a/templates/stream.html +++ b/templates/stream.html @@ -1,5 +1,6 @@ {% extends "layout.html" %} {% import 'utils.html' as utils %} +{% block title %}Stream - {{ config.NAME }}{% endblock %} {% block content %}
    {% include "header.html" %} diff --git a/templates/tags.html b/templates/tags.html index fc02452..0df993c 100644 --- a/templates/tags.html +++ b/templates/tags.html @@ -1,5 +1,6 @@ {% extends "layout.html" %} {% import 'utils.html' as utils %} +{% block title %}#{{ tag }} - {{ config.NAME }}{% endblock %} {% block header %} From 694511121cd5201de937719e122400e88fe216e3 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 21 May 2018 21:10:21 +0200 Subject: [PATCH 0035/1425] Add theme color config item --- config.py | 1 + templates/layout.html | 1 + 2 files changed, 2 insertions(+) diff --git a/config.py b/config.py index 250916c..12f0ae9 100644 --- a/config.py +++ b/config.py @@ -44,6 +44,7 @@ with open('config/me.yml') as f: ICON_URL = conf['icon_url'] PASS = conf['pass'] PUBLIC_INSTANCES = conf.get('public_instances') + THEME_COLOR = conf.get('theme_color') USER_AGENT = ( f'{requests.utils.default_user_agent()} ' diff --git a/templates/layout.html b/templates/layout.html index 648c06b..f1cc5b0 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -15,6 +15,7 @@ +{% if config.THEME_COLOR %}{% endif %}
    From cc900a2b4c0137bea1979fb4eb05e277b6a57f20 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 23 May 2018 00:41:37 +0200 Subject: [PATCH 0036/1425] Improve logging --- Dockerfile | 2 +- Makefile | 5 +++++ app.py | 6 ++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ae59305..10bf4ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,4 +3,4 @@ ADD . /app WORKDIR /app RUN pip install -r requirements.txt ENV FLASK_APP=app.py -CMD ["gunicorn", "-w", "2", "-b", "0.0.0.0:5005", "app:app"] +CMD ["gunicorn", "-w", "2", "-b", "0.0.0.0:5005", "--log-level", "debug", "app:app"] diff --git a/Makefile b/Makefile index bc473fc..333f991 100644 --- a/Makefile +++ b/Makefile @@ -3,3 +3,8 @@ css: password: python -c "import bcrypt; from getpass import getpass; print(bcrypt.hashpw(getpass().encode('utf-8'), bcrypt.gensalt()).decode('utf-8'))" + +update: + docker-compose stop + git pull + docker-compose up -d --force-recreate --build diff --git a/app.py b/app.py index e521e2a..fe0c444 100644 --- a/app.py +++ b/app.py @@ -57,6 +57,12 @@ from utils.webfinger import get_actor_url app = Flask(__name__) app.secret_key = get_secret_key('flask') +# Hook up Flask logging with gunicorn +gunicorn_logger = logging.getLogger('gunicorn.error') +root_logger = logging.getLogger() +root_logger.handlers = gunicorn_logger.handlers +root_logger.setLevel(gunicorn_logger.level) + JWT_SECRET = get_secret_key('jwt') JWT = JSONWebSignatureSerializer(JWT_SECRET) From d90e489fc6f7abbab7c64f80b8d6851c08e9fa1d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 23 May 2018 00:57:34 +0200 Subject: [PATCH 0037/1425] Improve the debug mode --- README.md | 2 +- docker-compose-tests.yml | 1 + utils/__init__.py | 12 ++++++++++++ utils/urlutils.py | 10 +++++++++- 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c93a814..9c96c79 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ $ pip install -r requirements.txt # Start the Celery worker, RabbitMQ and MongoDB $ docker-compose -f docker-compose-dev.yml up -d # Run the server locally -$ FLASK_APP=app.py flask run -p 5005 --with-threads +$ MICROBLOGPUB_DEBUG=1 FLASK_APP=app.py flask run -p 5005 --with-threads ``` ## Contributions diff --git a/docker-compose-tests.yml b/docker-compose-tests.yml index d41baf8..af55d05 100644 --- a/docker-compose-tests.yml +++ b/docker-compose-tests.yml @@ -13,6 +13,7 @@ services: environment: - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// - MICROBLOGPUB_MONGODB_HOST=mongo:27017 + - MICROBLOGPUB_DEBUG=1 celery: build: . links: diff --git a/utils/__init__.py b/utils/__init__.py index e69de29..c30c37d 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -0,0 +1,12 @@ +import logging + +logger = logging.getLogger(__name__) + + +def strtobool(s: str) -> bool: + if s in ['y', 'yes', 'true', 'on', '1']: + return True + if s in ['n', 'no', 'false', 'off', '0']: + return False + + raise ValueError(f'cannot convert {s} to bool') diff --git a/utils/urlutils.py b/utils/urlutils.py index b304f5e..3e9087e 100644 --- a/utils/urlutils.py +++ b/utils/urlutils.py @@ -1,16 +1,24 @@ import logging +import os import socket import ipaddress from urllib.parse import urlparse +from . import strtobool + logger = logging.getLogger(__name__) -def is_url_valid(url): +def is_url_valid(url: str) -> bool: parsed = urlparse(url) if parsed.scheme not in ['http', 'https']: return False + # XXX in debug mode, we want to allow requests to localhost to test the federation with local instances + debug_mode = strtobool(os.getenv('MICROBLOGPUB_DEBUG', 'false')) + if debug_mode: + return True + if parsed.hostname in ['localhost']: return False From 06f4f824d89251c5c2aa474f93ec790a587d08ae Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 25 May 2018 23:57:29 +0200 Subject: [PATCH 0038/1425] Tweak the webfinger resolution --- README.md | 2 ++ config.py | 1 + utils/urlutils.py | 11 ++++++++ utils/webfinger.py | 68 ++++++++++++++++++++++++++-------------------- 4 files changed, 52 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 9c96c79..d0a34da 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,8 @@ - Privacy-aware image upload endpoint that strip EXIF meta data before storing the file - No JavaScript, that's it, even the admin UI is pure HTML/CSS - Easy to customize (the theme is written Sass) + - mobile-friendly theme + - with dark and light version - Microformats aware (exports `h-feed`, `h-entry`, `h-cards`, ...) - Exports RSS/Atom feeds - Comes with a tiny HTTP API to help posting new content and performing basic actions diff --git a/config.py b/config.py index 12f0ae9..6002350 100644 --- a/config.py +++ b/config.py @@ -44,6 +44,7 @@ with open('config/me.yml') as f: ICON_URL = conf['icon_url'] PASS = conf['pass'] PUBLIC_INSTANCES = conf.get('public_instances') + # TODO(tsileo): choose dark/light style THEME_COLOR = conf.get('theme_color') USER_AGENT = ( diff --git a/utils/urlutils.py b/utils/urlutils.py index 3e9087e..be37c99 100644 --- a/utils/urlutils.py +++ b/utils/urlutils.py @@ -9,6 +9,10 @@ from . import strtobool logger = logging.getLogger(__name__) +class InvalidURLError(Exception): + pass + + def is_url_valid(url: str) -> bool: parsed = urlparse(url) if parsed.scheme not in ['http', 'https']: @@ -33,3 +37,10 @@ def is_url_valid(url: str) -> bool: return False return True + + +def check_url(url: str) -> None: + if not is_url_valid(url): + raise InvalidURLError(f'"{url}" is invalid') + + return None diff --git a/utils/webfinger.py b/utils/webfinger.py index 296ecae..93f33df 100644 --- a/utils/webfinger.py +++ b/utils/webfinger.py @@ -1,15 +1,25 @@ -from typing import Optional from urllib.parse import urlparse +from typing import Dict, Any +from typing import Optional +import logging import requests -def get_remote_follow_template(resource: str) -> Optional[str]: - """Mastodon-like WebFinger resolution to retrieve the activity stream Actor URL. +from .urlutils import check_url - Returns: - the Actor URL or None if the resolution failed. + +logger = logging.getLogger(__name__) + + +def webfinger(resource: str) -> Optional[Dict[str, Any]]: + """Mastodon-like WebFinger resolution to retrieve the activity stream Actor URL. """ - if resource.startswith('http'): + logger.info(f'performing webfinger resolution for {resource}') + protos = ['https', 'http'] + if resource.startswith('http://'): + protos.reverse() + host = urlparse(resource).netloc + elif resource.startswith('https://'): host = urlparse(resource).netloc else: if resource.startswith('acct:'): @@ -18,15 +28,30 @@ def get_remote_follow_template(resource: str) -> Optional[str]: resource = resource[1:] _, host = resource.split('@', 1) resource='acct:'+resource - resp = requests.get( - f'https://{host}/.well-known/webfinger', - {'resource': resource} - ) - print(resp, resp.request.url) + + # Security check on the url (like not calling localhost) + check_url(f'https://{host}') + + for i, proto in enumerate(protos): + try: + url = f'{proto}://{host}/.well-known/webfinger' + resp = requests.get( + url, + {'resource': resource} + ) + except requests.ConnectionError: + # If we tried https first and the domain is "http only" + if i == 0: + continue + break if resp.status_code == 404: return None resp.raise_for_status() - data = resp.json() + return resp.json() + + +def get_remote_follow_template(resource: str) -> Optional[str]: + data = webfinger(resource) for link in data['links']: if link.get('rel') == 'http://ostatus.org/schema/1.0/subscribe': return link.get('template') @@ -39,24 +64,7 @@ def get_actor_url(resource: str) -> Optional[str]: Returns: the Actor URL or None if the resolution failed. """ - if resource.startswith('http'): - host = urlparse(resource).netloc - else: - if resource.startswith('acct:'): - resource = resource[5:] - if resource.startswith('@'): - resource = resource[1:] - _, host = resource.split('@', 1) - resource='acct:'+resource - resp = requests.get( - f'https://{host}/.well-known/webfinger', - {'resource': resource} - ) - print(resp, resp.request.url) - if resp.status_code == 404: - return None - resp.raise_for_status() - data = resp.json() + data = webfinger(resource) for link in data['links']: if link.get('rel') == 'self' and link.get('type') == 'application/activity+json': return link.get('href') From a3267971e828d7d018877366bd9299db399e1267 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 26 May 2018 00:03:30 +0200 Subject: [PATCH 0039/1425] More url checking --- utils/actor_service.py | 5 +++++ utils/object_service.py | 3 +++ utils/opengraph.py | 3 ++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/utils/actor_service.py b/utils/actor_service.py index af7acc8..6c1f35a 100644 --- a/utils/actor_service.py +++ b/utils/actor_service.py @@ -4,6 +4,8 @@ import requests from urllib.parse import urlparse from Crypto.PublicKey import RSA +from .urlutils import check_url + logger = logging.getLogger(__name__) @@ -23,6 +25,9 @@ class ActorService(object): def _fetch(self, actor_url): logger.debug(f'fetching remote object {actor_url}') + + check_url(actor_url) + resp = requests.get(actor_url, headers={ 'Accept': 'application/activity+json', 'User-Agent': self._user_agent, diff --git a/utils/object_service.py b/utils/object_service.py index 185488f..9445550 100644 --- a/utils/object_service.py +++ b/utils/object_service.py @@ -1,6 +1,8 @@ import requests from urllib.parse import urlparse +from .urlutils import check_url + class ObjectService(object): def __init__(self, user_agent, col, inbox, outbox, instances): @@ -13,6 +15,7 @@ class ObjectService(object): def _fetch_remote(self, object_id): print(f'fetch remote {object_id}') + check_url(object_id) resp = requests.get(object_id, headers={ 'Accept': 'application/activity+json', 'User-Agent': self._user_agent, diff --git a/utils/opengraph.py b/utils/opengraph.py index e67a5a8..a53c07b 100644 --- a/utils/opengraph.py +++ b/utils/opengraph.py @@ -5,7 +5,7 @@ import opengraph import requests from bs4 import BeautifulSoup -from .urlutils import is_url_valid +from .urlutils import is_url_valid, check_url def links_from_note(note): @@ -38,6 +38,7 @@ def fetch_og_metadata(user_agent, col, remote_id): # FIXME(tsileo): set the user agent by giving HTML directly to OpenGraph htmls = [] for l in links: + check_url(l) r = requests.get(l, headers={'User-Agent': user_agent}) r.raise_for_status() htmls.append(r.text) From 6eb5d24f3eeb88eb041622b3cae3366729c9561f Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 26 May 2018 00:09:57 +0200 Subject: [PATCH 0040/1425] Fix mypy issue --- utils/webfinger.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils/webfinger.py b/utils/webfinger.py index 93f33df..8e6fdc7 100644 --- a/utils/webfinger.py +++ b/utils/webfinger.py @@ -52,6 +52,8 @@ def webfinger(resource: str) -> Optional[Dict[str, Any]]: def get_remote_follow_template(resource: str) -> Optional[str]: data = webfinger(resource) + if data is None: + return None for link in data['links']: if link.get('rel') == 'http://ostatus.org/schema/1.0/subscribe': return link.get('template') @@ -65,6 +67,8 @@ def get_actor_url(resource: str) -> Optional[str]: the Actor URL or None if the resolution failed. """ data = webfinger(resource) + if data is None: + return None for link in data['links']: if link.get('rel') == 'self' and link.get('type') == 'application/activity+json': return link.get('href') From fe46cb4317c92b193951dc661be975f8a1e2b136 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 26 May 2018 09:50:59 +0200 Subject: [PATCH 0041/1425] Add basic "federation" tests and some bugfixes --- .travis.yml | 5 +++- app.py | 4 +++ config.py | 2 +- docker-compose-tests.yml | 5 +++- tests/federation_test.py | 41 ++++++++++++++++++++++++++ tests/fixtures/instance1/config/me.yml | 6 ++-- tests/fixtures/instance2/config/me.yml | 6 ++-- 7 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 tests/federation_test.py diff --git a/.travis.yml b/.travis.yml index af5b21a..9c1a84f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,4 +21,7 @@ script: - docker-compose -p instance1 -f docker-compose-tests.yml ps - WEB_PORT=5007 CONFIG_DIR=./tests/fixtures/instance2/config docker-compose -p instance2 -f docker-compose-tests.yml up -d - docker-compose -p instance2 -f docker-compose-tests.yml ps - - pytest -v --ignore data + # Integration tests first + - pytest -v --ignore data -k integration + # Federation tests (with two local instances) + - pytest -v -s --ignore data -k federation diff --git a/app.py b/app.py index fe0c444..db0d98c 100644 --- a/app.py +++ b/app.py @@ -760,6 +760,8 @@ def api_upload(): @api_required def api_new_note(): source = request.args.get('content') + if not source: + raise ValueError('missing content') content, tags = parse_markdown(source) to = request.args.get('to') cc = [ID+'/followers'] @@ -792,6 +794,8 @@ def api_stream(): @api_required def api_follow(): actor = request.args.get('actor') + if not actor: + raise ValueError('missing actor') if DB.following.find({'remote_actor': actor}).count() > 0: return Response(status=201) diff --git a/config.py b/config.py index 6002350..6cb17e6 100644 --- a/config.py +++ b/config.py @@ -43,7 +43,7 @@ with open('config/me.yml') as f: SUMMARY = conf['summary'] ICON_URL = conf['icon_url'] PASS = conf['pass'] - PUBLIC_INSTANCES = conf.get('public_instances') + PUBLIC_INSTANCES = conf.get('public_instances', []) # TODO(tsileo): choose dark/light style THEME_COLOR = conf.get('theme_color') diff --git a/docker-compose-tests.yml b/docker-compose-tests.yml index af55d05..d45609b 100644 --- a/docker-compose-tests.yml +++ b/docker-compose-tests.yml @@ -1,4 +1,4 @@ -version: '3' +version: '3.5' services: web: build: . @@ -31,3 +31,6 @@ services: environment: - RABBITMQ_ERLANG_COOKIE=secretrabbit - RABBITMQ_NODENAME=rabbit@my-rabbit +networks: + default: + name: microblogpubfede diff --git a/tests/federation_test.py b/tests/federation_test.py new file mode 100644 index 0000000..f345dd0 --- /dev/null +++ b/tests/federation_test.py @@ -0,0 +1,41 @@ +import os + +import requests +from html2text import html2text + + +def resp2plaintext(resp): + """Convert the body of a requests reponse to plain text in order to make basic assertions.""" + return html2text(resp.text) + + +def test_federation(): + """Ensure the homepage is accessible.""" + resp = requests.get('http://localhost:5006') + resp.raise_for_status() + assert resp.status_code == 200 + + resp = requests.get('http://localhost:5007') + resp.raise_for_status() + assert resp.status_code == 200 + + # Keep one session per instance + + # Login + session1 = requests.Session() + resp = session1.post('http://localhost:5006/login', data={'pass': 'hello'}) + assert resp.status_code == 200 + + # Login + session2 = requests.Session() + resp = session2.post('http://localhost:5007/login', data={'pass': 'hello'}) + assert resp.status_code == 200 + + # Instance1 follows instance2 + resp = session1.get('http://localhost:5006/api/follow', params={'actor': 'http://instance2_web_1:5005'}) + assert resp.status_code == 201 + + resp = requests.get('http://localhost:5007/followers', headers={'Accept': 'application/activity+json'}) + resp.raise_for_status() + + assert resp.json()['first']['orderedItems'] == ['http://instance1_web_1:5005'] diff --git a/tests/fixtures/instance1/config/me.yml b/tests/fixtures/instance1/config/me.yml index 2513938..f7c803c 100644 --- a/tests/fixtures/instance1/config/me.yml +++ b/tests/fixtures/instance1/config/me.yml @@ -1,7 +1,7 @@ -username: 'instance_1' +username: 'instance1' name: 'Instance 1' icon_url: 'https://sos-ch-dk-2.exo.io/microblogpub/microblobpub.png' -domain: 'localhost:5006' -summary: 'instance 1 summary' +domain: 'instance1_web_1:5005' +summary: 'instance1 summary' pass: '$2b$12$nEgJMgaYbXSPOvgnqM4jSeYnleKhXqsFgv/o3hg12x79uEdsR4cUy' # hello https: false diff --git a/tests/fixtures/instance2/config/me.yml b/tests/fixtures/instance2/config/me.yml index 844e5c8..22f4127 100644 --- a/tests/fixtures/instance2/config/me.yml +++ b/tests/fixtures/instance2/config/me.yml @@ -1,7 +1,7 @@ -username: 'instance_2' +username: 'instance2' name: 'Instance 2' icon_url: 'https://sos-ch-dk-2.exo.io/microblogpub/microblobpub.png' -domain: 'localhost:5007' -summary: 'instance 2 summary' +domain: 'instance2_web_1:5005' +summary: 'instance2 summary' pass: '$2b$12$nEgJMgaYbXSPOvgnqM4jSeYnleKhXqsFgv/o3hg12x79uEdsR4cUy' # hello https: false From 849c495b7d4df1a1c2c538a97425525daa3048d3 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 26 May 2018 09:57:27 +0200 Subject: [PATCH 0042/1425] Fix tests --- tests/federation_test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/federation_test.py b/tests/federation_test.py index f345dd0..b77f8d9 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -1,3 +1,4 @@ +import time import os import requests @@ -35,7 +36,11 @@ def test_federation(): resp = session1.get('http://localhost:5006/api/follow', params={'actor': 'http://instance2_web_1:5005'}) assert resp.status_code == 201 + + time.sleep(2) resp = requests.get('http://localhost:5007/followers', headers={'Accept': 'application/activity+json'}) resp.raise_for_status() + print(resp.json()) + assert resp.json()['first']['orderedItems'] == ['http://instance1_web_1:5005'] From 3f09bee39c69a655b97d5a837d2923bd73f1af10 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 26 May 2018 10:02:24 +0200 Subject: [PATCH 0043/1425] Tweak tests --- tests/federation_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/federation_test.py b/tests/federation_test.py index b77f8d9..12bba0a 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -36,8 +36,7 @@ def test_federation(): resp = session1.get('http://localhost:5006/api/follow', params={'actor': 'http://instance2_web_1:5005'}) assert resp.status_code == 201 - - time.sleep(2) + time.sleep(10) resp = requests.get('http://localhost:5007/followers', headers={'Accept': 'application/activity+json'}) resp.raise_for_status() From 888410d646984edb44eee4565d19fa05aa5188a3 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 26 May 2018 10:43:05 +0200 Subject: [PATCH 0044/1425] Tests cleanup --- README.md | 7 +++- tests/federation_test.py | 78 +++++++++++++++++++++++++++------------- 2 files changed, 59 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index d0a34da..445cf6a 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,12 @@ - Easy to "cache" (the external/public-facing microblog part) - With a good setup, cached content can be served most of the time - You can setup a "purge" hook to let you invalidate cache when the microblog was updated - - Deployable with Docker + - Deployable with Docker (Docker compose for everything: dev, test and deployment) + - Focus on testing + - Tested against the [official ActivityPub test suite](https://test.activitypub.rocks/) ([ ] TODO submit the report) + - CI runs some local "federation" tests + - Manually tested against [Mastodon](https://github.com/tootsuite/mastodon) + - Project is running an up-to-date instance ## Running your instance diff --git a/tests/federation_test.py b/tests/federation_test.py index 12bba0a..20203de 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -10,36 +10,64 @@ def resp2plaintext(resp): return html2text(resp.text) +class Instance(object): + """Test instance wrapper.""" + + def __init__(self, host_url, docker_url=None): + self.host_url = host_url + self.docker_url = docker_url or host_url + self.session = requests.Session() + + def ping(self): + """Ensures the homepage is reachable.""" + resp = self.session.get(f'{self.host_url}/') + resp.raise_for_status() + assert resp.status_code == 200 + + def login(self): + resp = self.session.post(f'{self.host_url}/login', data={'pass': 'hello'}) + resp.raise_for_status() + assert resp.status_code == 200 + + def follow(self, instance: 'Instance') -> None: + # Instance1 follows instance2 + resp = self.session.get(f'{self.host_url}/api/follow', params={'actor': instance.docker_url}) + assert resp.status_code == 201 + + # We need to wait for the Follow/Accept dance + time.sleep(10) + + def followers(self): + resp = self.session.get(f'{self.host_url}/followers', headers={'Accept': 'application/activity+json'}) + resp.raise_for_status() + + data = resp.json() + + return resp.json()['first']['orderedItems'] + + def following(self): + resp = self.session.get(f'{self.host_url}/following', headers={'Accept': 'application/activity+json'}) + resp.raise_for_status() + + data = resp.json() + + return resp.json()['first']['orderedItems'] + + def test_federation(): """Ensure the homepage is accessible.""" - resp = requests.get('http://localhost:5006') - resp.raise_for_status() - assert resp.status_code == 200 + instance1 = Instance('http://localhost:5006', 'http://instance1_web_1:5005') + instance1.ping() - resp = requests.get('http://localhost:5007') - resp.raise_for_status() - assert resp.status_code == 200 - - # Keep one session per instance + instance2 = Instance('http://localhost:5007', 'http://instance2_web_1:5005') + instance2.ping() # Login - session1 = requests.Session() - resp = session1.post('http://localhost:5006/login', data={'pass': 'hello'}) - assert resp.status_code == 200 - - # Login - session2 = requests.Session() - resp = session2.post('http://localhost:5007/login', data={'pass': 'hello'}) - assert resp.status_code == 200 + instance1.login() + instance2.login() # Instance1 follows instance2 - resp = session1.get('http://localhost:5006/api/follow', params={'actor': 'http://instance2_web_1:5005'}) - assert resp.status_code == 201 + instance1.follow(instance2) - time.sleep(10) - resp = requests.get('http://localhost:5007/followers', headers={'Accept': 'application/activity+json'}) - resp.raise_for_status() - - print(resp.json()) - - assert resp.json()['first']['orderedItems'] == ['http://instance1_web_1:5005'] + assert instance2.followers() == [instance1.docker_url] + assert instance1.following() == [instance2.docker_url] From 25a75a9cef2bed500437b610d812f7148813bafa Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 27 May 2018 11:01:34 +0200 Subject: [PATCH 0045/1425] Cleanup, improve the collection resolver --- README.md | 8 +++++ activitypub.py | 43 ++----------------------- config.py | 4 +++ tests/federation_test.py | 5 +++ utils/activitypub_utils.py | 65 ++++++++++++++++++++++++++++++++++++++ utils/errors.py | 15 +++++++++ 6 files changed, 99 insertions(+), 41 deletions(-) create mode 100644 utils/activitypub_utils.py create mode 100644 utils/errors.py diff --git a/README.md b/README.md index 445cf6a..5784e93 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,14 @@ - Manually tested against [Mastodon](https://github.com/tootsuite/mastodon) - Project is running an up-to-date instance +## ActivityPub + +microblog.pub implements an [ActivityPub](http://activitypub.rocks/) server, it implements both the client to server API and the federated server to server API. + +Compatible with [Mastodon](https://github.com/tootsuite/mastodon) (which is not following the spec closely), but will drop OStatus messages. + +Activities are verified using HTTP Signatures or by fetching the content on the remote server directly. + ## Running your instance ### Installation diff --git a/activitypub.py b/activitypub.py index 5cf1bee..f1de0a8 100644 --- a/activitypub.py +++ b/activitypub.py @@ -4,13 +4,13 @@ import os from datetime import datetime from enum import Enum -import requests from bson.objectid import ObjectId from html2text import html2text from feedgen.feed import FeedGenerator from utils.linked_data_sig import generate_signature from utils.actor_service import NotAnActorError +from utils import activitypub_utils from config import USERNAME, BASE_URL, ID from config import CTX_AS, CTX_SECURITY, AS_PUBLIC from config import KEY, DB, ME, ACTOR_SERVICE @@ -936,46 +936,7 @@ def parse_collection(payload: Optional[Dict[str, Any]] = None, url: Optional[str return [doc['remote_actor'] for doc in DB.following.find()] # Go through all the pages - out: List[str] = [] - if url: - resp = requests.get(url, headers={'Accept': 'application/activity+json'}) - resp.raise_for_status() - payload = resp.json() - - if not payload: - raise ValueError('must at least prove a payload or an URL') - - if payload['type'] in ['Collection', 'OrderedCollection']: - if 'orderedItems' in payload: - return payload['orderedItems'] - if 'items' in payload: - return payload['items'] - if 'first' in payload: - if 'orderedItems' in payload['first']: - out.extend(payload['first']['orderedItems']) - if 'items' in payload['first']: - out.extend(payload['first']['items']) - n = payload['first'].get('next') - if n: - out.extend(parse_collection(url=n)) - return out - - while payload: - if payload['type'] in ['CollectionPage', 'OrderedCollectionPage']: - if 'orderedItems' in payload: - out.extend(payload['orderedItems']) - if 'items' in payload: - out.extend(payload['items']) - n = payload.get('next') - if n is None: - break - resp = requests.get(n, headers={'Accept': 'application/activity+json'}) - resp.raise_for_status() - payload = resp.json() - else: - raise Exception('unexpected activity type {}'.format(payload['type'])) - - return out + return activitypub_utils.parse_collection(payload, url) def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, col_name=None): diff --git a/config.py b/config.py index 6cb17e6..5bbebe5 100644 --- a/config.py +++ b/config.py @@ -4,6 +4,7 @@ import yaml from pymongo import MongoClient import requests +from utils import strtobool from utils.key import Key from utils.actor_service import ActorService from utils.object_service import ObjectService @@ -20,6 +21,9 @@ except ModuleNotFoundError: VERSION = subprocess.check_output(['git', 'describe', '--always']).split()[0].decode('utf-8') +DEBUG_MODE = strtobool(os.getenv('MICROBLOGPUB_DEBUG', 'false')) + + CTX_AS = 'https://www.w3.org/ns/activitystreams' CTX_SECURITY = 'https://w3id.org/security/v1' AS_PUBLIC = 'https://www.w3.org/ns/activitystreams#Public' diff --git a/tests/federation_test.py b/tests/federation_test.py index 20203de..f291638 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -53,6 +53,11 @@ class Instance(object): return resp.json()['first']['orderedItems'] + def outbox(self): + resp = self.session.get(f'{self.host_url}/following', headers={'Accept': 'application/activity+json'}) + resp.raise_for_status() + return resp.json() + def test_federation(): """Ensure the homepage is accessible.""" diff --git a/utils/activitypub_utils.py b/utils/activitypub_utils.py new file mode 100644 index 0000000..0275f54 --- /dev/null +++ b/utils/activitypub_utils.py @@ -0,0 +1,65 @@ +from typing import Optional, Dict, List, Any + +import requests + +from .errors import RecursionLimitExceededError +from .errors import UnexpectedActivityTypeError + + +def _do_req(url: str, headers: Dict[str, str]) -> Dict[str, Any]: + resp = requests.get(url, headers=headers) + resp.raise_for_status() + return resp.json() + + +def parse_collection( + payload: Optional[Dict[str, Any]] = None, + url: Optional[str] = None, + user_agent: Optional[str] = None, + level: int = 0, + do_req: Any = _do_req, +) -> List[str]: + """Resolve/fetch a `Collection`/`OrderedCollection`.""" + if level > 3: + raise RecursionLimitExceededError('recursion limit exceeded') + + # Go through all the pages + headers = {'Accept': 'application/activity+json'} + if user_agent: + headers['User-Agent'] = user_agent + + out: List[str] = [] + if url: + payload = do_req(url, headers) + if not payload: + raise ValueError('must at least prove a payload or an URL') + + if payload['type'] in ['Collection', 'OrderedCollection']: + if 'orderedItems' in payload: + return payload['orderedItems'] + if 'items' in payload: + return payload['items'] + if 'first' in payload: + if 'orderedItems' in payload['first']: + out.extend(payload['first']['orderedItems']) + if 'items' in payload['first']: + out.extend(payload['first']['items']) + n = payload['first'].get('next') + if n: + out.extend(parse_collection(url=n, user_agent=user_agent, level=level+1, do_req=do_req)) + return out + + while payload: + if payload['type'] in ['CollectionPage', 'OrderedCollectionPage']: + if 'orderedItems' in payload: + out.extend(payload['orderedItems']) + if 'items' in payload: + out.extend(payload['items']) + n = payload.get('next') + if n is None: + break + payload = do_req(n, headers) + else: + raise UnexpectedActivityTypeError('unexpected activity type {}'.format(payload['type'])) + + return out diff --git a/utils/errors.py b/utils/errors.py new file mode 100644 index 0000000..31e678e --- /dev/null +++ b/utils/errors.py @@ -0,0 +1,15 @@ + +class Error(Exception): + pass + + +class BadActivityError(Error): + pass + + +class RecursionLimitExceededError(BadActivityError): + pass + + +class UnexpectedActivityTypeError(BadActivityError): + pass From cf242b2d846645ba6f2fff801b146cd9a77ba130 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 27 May 2018 11:50:09 +0200 Subject: [PATCH 0046/1425] Improve the tests, add a new debug endpoint --- .travis.yml | 4 ++-- app.py | 20 ++++++++++++++++++++ config.py | 10 +++++++++- tests/federation_test.py | 21 +++++++++++++++++++++ 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9c1a84f..8477299 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,6 @@ script: - WEB_PORT=5007 CONFIG_DIR=./tests/fixtures/instance2/config docker-compose -p instance2 -f docker-compose-tests.yml up -d - docker-compose -p instance2 -f docker-compose-tests.yml ps # Integration tests first - - pytest -v --ignore data -k integration + - python -m pytest -v --ignore data -k integration # Federation tests (with two local instances) - - pytest -v -s --ignore data -k federation + - python -m pytest -v -s --ignore data -k federation diff --git a/app.py b/app.py index db0d98c..0b19ef0 100644 --- a/app.py +++ b/app.py @@ -21,6 +21,7 @@ from flask import redirect from flask import Response from flask import render_template from flask import session +from flask import jsonify as flask_jsonify from flask import url_for from html2text import html2text from itsdangerous import JSONWebSignatureSerializer @@ -47,6 +48,8 @@ from config import OBJECT_SERVICE from config import PASS from config import HEADERS from config import VERSION +from config import DEBUG_MODE +from config import _drop_db from config import custom_cache_purge_hook from utils.httpsig import HTTPSigAuth, verify_request from utils.key import get_secret_key @@ -710,6 +713,23 @@ def inbox(): ) +@app.route('/api/debug', methods=['GET', 'DELETE']) +@api_required +def api_debug(): + """Endpoint used/needed for testing, only works in DEBUG_MODE.""" + if not DEBUG_MODE: + return flask_jsonify(message='DEBUG_MODE is off') + + if request.method == 'DELETE': + _drop_db() + return flask_jsonify(message='DB dropped') + + return flask_jsonify( + inbox=DB.inbox.count(), + outbox=DB.outbox.count(), + ) + + @app.route('/api/upload', methods=['POST']) @api_required def api_upload(): diff --git a/config.py b/config.py index 5bbebe5..e4c3912 100644 --- a/config.py +++ b/config.py @@ -62,7 +62,15 @@ mongo_client = MongoClient( host=[os.getenv('MICROBLOGPUB_MONGODB_HOST', 'localhost:27017')], ) -DB = mongo_client['{}_{}'.format(USERNAME, DOMAIN.replace('.', '_'))] +DB_NAME = '{}_{}'.format(USERNAME, DOMAIN.replace('.', '_')) +DB = mongo_client[DB_NAME] + +def _drop_db(): + if not DEBUG_MODE: + return + + mongo_client.drop_database(DB_NAME) + KEY = Key(USERNAME, DOMAIN, create=True) ME = { diff --git a/tests/federation_test.py b/tests/federation_test.py index f291638..17cd747 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -24,6 +24,18 @@ class Instance(object): resp.raise_for_status() assert resp.status_code == 200 + def debug(self): + resp = self.session.get(f'{self.host_url}/api/debug', headers={'Accept': 'application/json'}) + resp.raise_for_status() + + return resp.json() + + def drop_db(self): + resp = self.session.delete(f'{self.host_url}/api/debug', headers={'Accept': 'application/json'}) + resp.raise_for_status() + + return resp.json() + def login(self): resp = self.session.post(f'{self.host_url}/login', data={'pass': 'hello'}) resp.raise_for_status() @@ -63,9 +75,11 @@ def test_federation(): """Ensure the homepage is accessible.""" instance1 = Instance('http://localhost:5006', 'http://instance1_web_1:5005') instance1.ping() + instance1.drop_db() instance2 = Instance('http://localhost:5007', 'http://instance2_web_1:5005') instance2.ping() + instance2.drop_db() # Login instance1.login() @@ -73,6 +87,13 @@ def test_federation(): # Instance1 follows instance2 instance1.follow(instance2) + instance1_debug = instance1.debug() + assert instance1_debug['inbox'] == 1 # An Accept activity should be there + assert instance1_debug['outbox'] == 1 # We've sent a Follow activity + + instance2_debug = instance2.debug() + assert instance1_debug['inbox'] == 1 # An Follow activity should be there + assert instance1_debug['outbox'] == 1 # We've sent a Accept activity assert instance2.followers() == [instance1.docker_url] assert instance1.following() == [instance2.docker_url] From 1e3d9279ee53a62e2139a2643395a0adad18ee0e Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 27 May 2018 11:52:27 +0200 Subject: [PATCH 0047/1425] Fix the test suite --- tests/federation_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/federation_test.py b/tests/federation_test.py index 17cd747..7614327 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -75,15 +75,15 @@ def test_federation(): """Ensure the homepage is accessible.""" instance1 = Instance('http://localhost:5006', 'http://instance1_web_1:5005') instance1.ping() - instance1.drop_db() instance2 = Instance('http://localhost:5007', 'http://instance2_web_1:5005') instance2.ping() - instance2.drop_db() # Login instance1.login() + instance1.drop_db() instance2.login() + instance2.drop_db() # Instance1 follows instance2 instance1.follow(instance2) From 166fc91c54e840401c2b9637c5632dbec1a31814 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 27 May 2018 12:02:14 +0200 Subject: [PATCH 0048/1425] Improve tests --- tests/federation_test.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/federation_test.py b/tests/federation_test.py index 7614327..dc3d17d 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -3,6 +3,7 @@ import os import requests from html2text import html2text +from utils import activitypub_utils def resp2plaintext(resp): @@ -18,6 +19,15 @@ class Instance(object): self.docker_url = docker_url or host_url self.session = requests.Session() + def _do_req(self, url, headers): + url = url.replace(self.docker_url, self.host_url) + resp = requests.get(url, headers=headers) + resp.raise_for_status() + return resp.json() + + def _parse_collection(self, payload=None, url=None): + return activitypub_utils.parse_collection(url=url, payload=payload, do_req=self._do_req) + def ping(self): """Ensures the homepage is reachable.""" resp = self.session.get(f'{self.host_url}/') @@ -55,7 +65,7 @@ class Instance(object): data = resp.json() - return resp.json()['first']['orderedItems'] + return self._parse_collection(payload=data) def following(self): resp = self.session.get(f'{self.host_url}/following', headers={'Accept': 'application/activity+json'}) @@ -63,7 +73,7 @@ class Instance(object): data = resp.json() - return resp.json()['first']['orderedItems'] + return self._parse_collection(payload=data) def outbox(self): resp = self.session.get(f'{self.host_url}/following', headers={'Accept': 'application/activity+json'}) From 12feb38a8faa6e36bfccf969b9ceaaf6cf5358a8 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 27 May 2018 14:21:06 +0200 Subject: [PATCH 0049/1425] More tests and bugfixes --- activitypub.py | 11 ++++++++ app.py | 20 +++++++++++---- tests/federation_test.py | 54 +++++++++++++++++++++++++++++++++++++--- 3 files changed, 76 insertions(+), 9 deletions(-) diff --git a/activitypub.py b/activitypub.py index f1de0a8..156a560 100644 --- a/activitypub.py +++ b/activitypub.py @@ -1,3 +1,4 @@ +import logging import json import binascii import os @@ -21,6 +22,8 @@ import tasks from typing import List, Optional, Dict, Any, Union from typing import TypeVar +logger = logging.getLogger(__name__) + A = TypeVar('A', bound='BaseActivity') ObjectType = Dict[str, Any] ObjectOrIDType = Union[str, ObjectType] @@ -454,6 +457,9 @@ class Follow(BaseActivity): def _undo_inbox(self) -> None: DB.followers.delete_one({'remote_actor': self.get_actor().id}) + def _undo_outbox(self) -> None: + DB.following.delete_one({'remote_actor': self.get_object().id}) + def build_accept(self) -> BaseActivity: return self._build_reply(ActivityTypes.ACCEPT) @@ -517,12 +523,17 @@ class Undo(BaseActivity): return False def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: + logger.debug('processing undo to outbox') + logger.debug('self={}'.format(self)) obj = self.get_object() + logger.debug('obj={}'.format(obj)) DB.outbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) try: obj._undo_outbox() + logger.debug(f'_undo_outbox called for {obj}') except NotImplementedError: + logger.debug(f'_undo_outbox not implemented for {obj}') pass diff --git a/app.py b/app.py index 0b19ef0..7eaf668 100644 --- a/app.py +++ b/app.py @@ -615,16 +615,25 @@ def ui_like(): like.post_to_outbox() return redirect(request.args.get('redirect')) -@app.route('/ui/undo') -@login_required -def ui_undo(): +@app.route('/api/undo', methods=['GET', 'POST']) +@api_required +def api_undo(): oid = request.args.get('id') - doc =DB.outbox.find_one({'id': oid}) + doc = DB.outbox.find_one({'$or': [{'id': oid}, {'remote_id': oid}]}) + undo_id = None if doc: obj = activitypub.parse_activity(doc.get('activity')) + # FIXME(tsileo): detect already undo-ed and make this API call idempotent undo = obj.build_undo() undo.post_to_outbox() - return redirect(request.args.get('redirect')) + undo_id = undo.id + if request.args.get('redirect'): + return redirect(request.args.get('redirect')) + return Response( + status=201, + headers={'Microblogpub-Created-Activity': undo_id}, + ) + @app.route('/stream') @login_required @@ -823,6 +832,7 @@ def api_follow(): follow.post_to_outbox() return Response( status=201, + headers={'Microblogpub-Created-Activity': follow.id}, ) diff --git a/tests/federation_test.py b/tests/federation_test.py index dc3d17d..ff8db17 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -58,6 +58,15 @@ class Instance(object): # We need to wait for the Follow/Accept dance time.sleep(10) + return resp.headers.get('microblogpub-created-activity') + + def undo(self, oid: str) -> None: + resp = self.session.get(f'{self.host_url}/api/undo', params={'id': oid}) + assert resp.status_code == 201 + + # We need to wait for the Follow/Accept dance + time.sleep(10) + return resp.headers.get('microblogpub-created-activity') def followers(self): resp = self.session.get(f'{self.host_url}/followers', headers={'Accept': 'application/activity+json'}) @@ -81,8 +90,7 @@ class Instance(object): return resp.json() -def test_federation(): - """Ensure the homepage is accessible.""" +def _instances(): instance1 = Instance('http://localhost:5006', 'http://instance1_web_1:5005') instance1.ping() @@ -94,16 +102,54 @@ def test_federation(): instance1.drop_db() instance2.login() instance2.drop_db() + + return instance1, instance2 + +def test_follow(): + instance1, instance2 = _instances() # Instance1 follows instance2 instance1.follow(instance2) instance1_debug = instance1.debug() + print(f'instance1_debug={instance1_debug}') + assert instance1_debug['inbox'] == 1 # An Accept activity should be there + assert instance1_debug['outbox'] == 1 # We've sent a Follow activity + + instance2_debug = instance2.debug() + print(f'instance2_debug={instance2_debug}') + assert instance2_debug['inbox'] == 1 # An Follow activity should be there + assert instance2_debug['outbox'] == 1 # We've sent a Accept activity + + assert instance2.followers() == [instance1.docker_url] + assert instance1.following() == [instance2.docker_url] + + +def test_follow_unfollow(): + instance1, instance2 = _instances() + # Instance1 follows instance2 + follow_id = instance1.follow(instance2) + instance1_debug = instance1.debug() assert instance1_debug['inbox'] == 1 # An Accept activity should be there assert instance1_debug['outbox'] == 1 # We've sent a Follow activity instance2_debug = instance2.debug() - assert instance1_debug['inbox'] == 1 # An Follow activity should be there - assert instance1_debug['outbox'] == 1 # We've sent a Accept activity + assert instance2_debug['inbox'] == 1 # An Follow activity should be there + assert instance2_debug['outbox'] == 1 # We've sent a Accept activity assert instance2.followers() == [instance1.docker_url] assert instance1.following() == [instance2.docker_url] + + instance1.undo(follow_id) + + assert instance2.followers() == [] + assert instance1.following() == [] + + instance1_debug = instance1.debug() + assert instance1_debug['inbox'] == 1 # An Accept activity should be there + assert instance1_debug['outbox'] == 2 # We've sent a Follow and a Undo activity + + instance2_debug = instance2.debug() + assert instance2_debug['inbox'] == 2 # An Follow and Undo activity should be there + assert instance2_debug['outbox'] == 1 # We've sent a Accept activity + + From 7a8621e72e423782d85efd16e537c3db2d2e111c Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 27 May 2018 20:40:42 +0200 Subject: [PATCH 0050/1425] Rename the ActivityType enum --- activitypub.py | 115 ++++++++++++++++++++++++------------------------- app.py | 17 ++++---- 2 files changed, 66 insertions(+), 66 deletions(-) diff --git a/activitypub.py b/activitypub.py index 156a560..353faca 100644 --- a/activitypub.py +++ b/activitypub.py @@ -24,12 +24,11 @@ from typing import TypeVar logger = logging.getLogger(__name__) -A = TypeVar('A', bound='BaseActivity') ObjectType = Dict[str, Any] ObjectOrIDType = Union[str, ObjectType] -class ActivityTypes(Enum): +class ActivityType(Enum): ANNOUNCE = 'Announce' BLOCK = 'Block' LIKE = 'Like' @@ -84,8 +83,8 @@ def _get_actor_id(actor: ObjectOrIDType) -> str: class BaseActivity(object): - ACTIVITY_TYPE: Optional[ActivityTypes] = None - ALLOWED_OBJECT_TYPES: List[ActivityTypes] = [] + ACTIVITY_TYPE: Optional[ActivityType] = None + ALLOWED_OBJECT_TYPES: List[ActivityType] = [] def __init__(self, **kwargs) -> None: if not self.ACTIVITY_TYPE: @@ -99,7 +98,7 @@ class BaseActivity(object): if 'id' in kwargs: self._data['id'] = kwargs.pop('id') - if self.ACTIVITY_TYPE != ActivityTypes.PERSON: + if self.ACTIVITY_TYPE != ActivityType.PERSON: actor = kwargs.get('actor') if actor: kwargs.pop('actor') @@ -117,9 +116,9 @@ class BaseActivity(object): else: if not self.ALLOWED_OBJECT_TYPES: raise ValueError('unexpected object') - if 'type' not in obj or (self.ACTIVITY_TYPE != ActivityTypes.CREATE and 'id' not in obj): + if 'type' not in obj or (self.ACTIVITY_TYPE != ActivityType.CREATE and 'id' not in obj): raise ValueError('invalid object') - if ActivityTypes(obj['type']) not in self.ALLOWED_OBJECT_TYPES: + if ActivityType(obj['type']) not in self.ALLOWED_OBJECT_TYPES: print(self, kwargs) raise ValueError(f'unexpected object type {obj["type"]} (allowed={self.ALLOWED_OBJECT_TYPES})') self._data['object'] = obj @@ -185,8 +184,8 @@ class BaseActivity(object): return self._data.get(name) @property - def type_enum(self) -> ActivityTypes: - return ActivityTypes(self.type) + def type_enum(self) -> ActivityType: + return ActivityType(self.type) def _set_id(self, uri: str, obj_id: str) -> None: raise NotImplementedError @@ -199,7 +198,7 @@ class BaseActivity(object): pass def _actor_id(self, obj: ObjectOrIDType) -> str: - if isinstance(obj, dict) and obj['type'] == ActivityTypes.PERSON.value: + if isinstance(obj, dict) and obj['type'] == ActivityType.PERSON.value: obj_id = obj.get('id') if not obj_id: raise ValueError('missing object id') @@ -223,11 +222,11 @@ class BaseActivity(object): if isinstance(self._data['object'], dict): p = parse_activity(self._data['object']) else: - if self.ACTIVITY_TYPE == ActivityTypes.FOLLOW: + if self.ACTIVITY_TYPE == ActivityType.FOLLOW: p = Person(**ACTOR_SERVICE.get(self._data['object'])) else: obj = OBJECT_SERVICE.get(self._data['object']) - if ActivityTypes(obj.get('type')) not in self.ALLOWED_OBJECT_TYPES: + if ActivityType(obj.get('type')) not in self.ALLOWED_OBJECT_TYPES: raise ValueError('invalid object type') p = parse_activity(obj) @@ -249,7 +248,7 @@ class BaseActivity(object): def get_actor(self) -> 'BaseActivity': actor = self._data.get('actor') if not actor: - if self.type_enum == ActivityTypes.NOTE: + if self.type_enum == ActivityType.NOTE: actor = str(self._data.get('attributedTo')) else: raise ValueError('failed to fetch actor') @@ -279,7 +278,7 @@ class BaseActivity(object): self.verify() actor = self.get_actor() - if DB.outbox.find_one({'type': ActivityTypes.BLOCK.value, + if DB.outbox.find_one({'type': ActivityType.BLOCK.value, 'activity.object': actor.id, 'meta.undo': False}): print('actor is blocked, drop activity') @@ -357,8 +356,8 @@ class BaseActivity(object): actor = Person(**ACTOR_SERVICE.get(recipient)) except NotAnActorError as error: # Is the activity a `Collection`/`OrderedCollection`? - if error.activity and error.activity['type'] in [ActivityTypes.COLLECTION.value, - ActivityTypes.ORDERED_COLLECTION.value]: + if error.activity and error.activity['type'] in [ActivityType.COLLECTION.value, + ActivityType.ORDERED_COLLECTION.value]: for item in parse_collection(error.activity): if item in [ME, AS_PUBLIC]: continue @@ -393,7 +392,7 @@ class BaseActivity(object): class Person(BaseActivity): - ACTIVITY_TYPE = ActivityTypes.PERSON + ACTIVITY_TYPE = ActivityType.PERSON def _init(self, **kwargs): # if 'icon' in kwargs: @@ -410,15 +409,15 @@ class Person(BaseActivity): class Block(BaseActivity): - ACTIVITY_TYPE = ActivityTypes.BLOCK + ACTIVITY_TYPE = ActivityType.BLOCK class Collection(BaseActivity): - ACTIVITY_TYPE = ActivityTypes.COLLECTION + ACTIVITY_TYPE = ActivityType.COLLECTION class Image(BaseActivity): - ACTIVITY_TYPE = ActivityTypes.IMAGE + ACTIVITY_TYPE = ActivityType.IMAGE NO_CONTEXT = True def _init(self, **kwargs): @@ -431,11 +430,11 @@ class Image(BaseActivity): class Follow(BaseActivity): - ACTIVITY_TYPE = ActivityTypes.FOLLOW - ALLOWED_OBJECT_TYPES = [ActivityTypes.PERSON] + ACTIVITY_TYPE = ActivityType.FOLLOW + ALLOWED_OBJECT_TYPES = [ActivityType.PERSON] - def _build_reply(self, reply_type: ActivityTypes) -> BaseActivity: - if reply_type == ActivityTypes.ACCEPT: + def _build_reply(self, reply_type: ActivityType) -> BaseActivity: + if reply_type == ActivityType.ACCEPT: return Accept( object=self.to_dict(embed=True), ) @@ -461,7 +460,7 @@ class Follow(BaseActivity): DB.following.delete_one({'remote_actor': self.get_object().id}) def build_accept(self) -> BaseActivity: - return self._build_reply(ActivityTypes.ACCEPT) + return self._build_reply(ActivityType.ACCEPT) def build_undo(self) -> BaseActivity: return Undo(object=self.to_dict(embed=True)) @@ -472,8 +471,8 @@ class Follow(BaseActivity): class Accept(BaseActivity): - ACTIVITY_TYPE = ActivityTypes.ACCEPT - ALLOWED_OBJECT_TYPES = [ActivityTypes.FOLLOW] + ACTIVITY_TYPE = ActivityType.ACCEPT + ALLOWED_OBJECT_TYPES = [ActivityType.FOLLOW] def _recipients(self) -> List[str]: return [self.get_object().get_actor().id] @@ -490,12 +489,12 @@ class Accept(BaseActivity): class Undo(BaseActivity): - ACTIVITY_TYPE = ActivityTypes.UNDO - ALLOWED_OBJECT_TYPES = [ActivityTypes.FOLLOW, ActivityTypes.LIKE, ActivityTypes.ANNOUNCE] + ACTIVITY_TYPE = ActivityType.UNDO + ALLOWED_OBJECT_TYPES = [ActivityType.FOLLOW, ActivityType.LIKE, ActivityType.ANNOUNCE] def _recipients(self) -> List[str]: obj = self.get_object() - if obj.type_enum == ActivityTypes.FOLLOW: + if obj.type_enum == ActivityType.FOLLOW: return [obj.get_object().id] else: return [obj.get_object().get_actor().id] @@ -538,8 +537,8 @@ class Undo(BaseActivity): class Like(BaseActivity): - ACTIVITY_TYPE = ActivityTypes.LIKE - ALLOWED_OBJECT_TYPES = [ActivityTypes.NOTE] + ACTIVITY_TYPE = ActivityType.LIKE + ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] def _recipients(self) -> List[str]: return [self.get_object().get_actor().id] @@ -577,8 +576,8 @@ class Like(BaseActivity): class Announce(BaseActivity): - ACTIVITY_TYPE = ActivityTypes.ANNOUNCE - ALLOWED_OBJECT_TYPES = [ActivityTypes.NOTE] + ACTIVITY_TYPE = ActivityType.ANNOUNCE + ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] def _recipients(self) -> List[str]: recipients = [] @@ -638,8 +637,8 @@ class Announce(BaseActivity): class Delete(BaseActivity): - ACTIVITY_TYPE = ActivityTypes.DELETE - ALLOWED_OBJECT_TYPES = [ActivityTypes.NOTE, ActivityTypes.TOMBSTONE] + ACTIVITY_TYPE = ActivityType.DELETE + ALLOWED_OBJECT_TYPES = [ActivityType.NOTE, ActivityType.TOMBSTONE] def _recipients(self) -> List[str]: return self.get_object().recipients() @@ -654,15 +653,15 @@ class Delete(BaseActivity): class Update(BaseActivity): - ACTIVITY_TYPE = ActivityTypes.UPDATE - ALLOWED_OBJECT_TYPES = [ActivityTypes.NOTE, ActivityTypes.PERSON] + ACTIVITY_TYPE = ActivityType.UPDATE + ALLOWED_OBJECT_TYPES = [ActivityType.NOTE, ActivityType.PERSON] # TODO(tsileo): ensure the actor updating is the same as the orinial activity # (ensuring that the Update and its object are of same origin) def _process_from_inbox(self): obj = self.get_object() - if obj.type_enum == ActivityTypes.NOTE: + if obj.type_enum == ActivityType.NOTE: DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'activity.object': obj.to_dict()}}) return @@ -694,8 +693,8 @@ class Update(BaseActivity): class Create(BaseActivity): - ACTIVITY_TYPE = ActivityTypes.CREATE - ALLOWED_OBJECT_TYPES = [ActivityTypes.NOTE] + ACTIVITY_TYPE = ActivityType.CREATE + ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] def _set_id(self, uri: str, obj_id: str) -> None: self._data['object']['id'] = uri + '/activity' @@ -768,11 +767,11 @@ class Create(BaseActivity): class Tombstone(BaseActivity): - ACTIVITY_TYPE = ActivityTypes.TOMBSTONE + ACTIVITY_TYPE = ActivityType.TOMBSTONE class Note(BaseActivity): - ACTIVITY_TYPE = ActivityTypes.NOTE + ACTIVITY_TYPE = ActivityType.NOTE def _init(self, **kwargs): print(self._data) @@ -831,25 +830,25 @@ class Note(BaseActivity): _ACTIVITY_TYPE_TO_CLS = { - ActivityTypes.IMAGE: Image, - ActivityTypes.PERSON: Person, - ActivityTypes.FOLLOW: Follow, - ActivityTypes.ACCEPT: Accept, - ActivityTypes.UNDO: Undo, - ActivityTypes.LIKE: Like, - ActivityTypes.ANNOUNCE: Announce, - ActivityTypes.UPDATE: Update, - ActivityTypes.DELETE: Delete, - ActivityTypes.CREATE: Create, - ActivityTypes.NOTE: Note, - ActivityTypes.BLOCK: Block, - ActivityTypes.COLLECTION: Collection, - ActivityTypes.TOMBSTONE: Tombstone, + ActivityType.IMAGE: Image, + ActivityType.PERSON: Person, + ActivityType.FOLLOW: Follow, + ActivityType.ACCEPT: Accept, + ActivityType.UNDO: Undo, + ActivityType.LIKE: Like, + ActivityType.ANNOUNCE: Announce, + ActivityType.UPDATE: Update, + ActivityType.DELETE: Delete, + ActivityType.CREATE: Create, + ActivityType.NOTE: Note, + ActivityType.BLOCK: Block, + ActivityType.COLLECTION: Collection, + ActivityType.TOMBSTONE: Tombstone, } def parse_activity(payload: ObjectType) -> BaseActivity: - t = ActivityTypes(payload['type']) + t = ActivityType(payload['type']) if t not in _ACTIVITY_TYPE_TO_CLS: raise ValueError('unsupported activity type') diff --git a/app.py b/app.py index 7eaf668..21a61a2 100644 --- a/app.py +++ b/app.py @@ -33,7 +33,7 @@ from werkzeug.utils import secure_filename import activitypub import config -from activitypub import ActivityTypes +from activitypub import ActivityType from activitypub import clean_activity from utils.content_helper import parse_markdown from config import KEY @@ -444,7 +444,7 @@ def outbox(): # FIXME(tsileo): filter deleted, add query support for build_ordered_collection q = { 'meta.deleted': False, - 'type': {'$in': [ActivityTypes.CREATE.value, ActivityTypes.ANNOUNCE.value]}, + 'type': {'$in': [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, } return jsonify(**activitypub.build_ordered_collection( DB.outbox, @@ -463,7 +463,7 @@ def outbox(): print(data) activity = activitypub.parse_activity(data) - if activity.type_enum == ActivityTypes.NOTE: + if activity.type_enum == ActivityType.NOTE: activity = activity.build_create() activity.post_to_outbox() @@ -486,7 +486,7 @@ def outbox_activity(item_id): if not data: abort(404) obj = data['activity'] - if obj['type'] != ActivityTypes.CREATE.value: + if obj['type'] != ActivityType.CREATE.value: abort(404) return jsonify(**clean_activity(obj['object'])) @@ -496,7 +496,7 @@ def admin(): q = { 'meta.deleted': False, 'meta.undo': False, - 'type': ActivityTypes.LIKE.value, + 'type': ActivityType.LIKE.value, } col_liked = DB.outbox.count(q) @@ -801,7 +801,7 @@ def api_new_note(): note = activitypub.Note( cc=cc, to=[to if to else config.AS_PUBLIC], - content=content, # TODO(tsileo): handle markdown + content=content, tag=tags, source={'mediaType': 'text/markdown', 'content': source}, ) @@ -810,6 +810,7 @@ def api_new_note(): return Response( status=201, response='OK', + headers={'Microblogpub-Created-Activity': created.id}, ) @app.route('/api/stream') @@ -895,7 +896,7 @@ def tags(tag): q = { 'meta.deleted': False, 'meta.undo': False, - 'type': ActivityTypes.CREATE.value, + 'type': ActivityType.CREATE.value, 'activity.object.tag.type': 'Hashtag', 'activity.object.tag.name': '#'+tag, } @@ -915,7 +916,7 @@ def liked(): q = { 'meta.deleted': False, 'meta.undo': False, - 'type': ActivityTypes.LIKE.value, + 'type': ActivityType.LIKE.value, } return jsonify(**activitypub.build_ordered_collection( DB.outbox, From 942d12a7c7ee4d2695e61faa6d268db77ad5c4a3 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 27 May 2018 22:30:43 +0200 Subject: [PATCH 0051/1425] More tests and bugfixes --- activitypub.py | 59 ++++++++++++++++++++++++++++------------ app.py | 3 +- tests/federation_test.py | 36 +++++++++++++++++++++--- 3 files changed, 75 insertions(+), 23 deletions(-) diff --git a/activitypub.py b/activitypub.py index 353faca..95ce35a 100644 --- a/activitypub.py +++ b/activitypub.py @@ -11,6 +11,7 @@ from feedgen.feed import FeedGenerator from utils.linked_data_sig import generate_signature from utils.actor_service import NotAnActorError +from utils.errors import BadActivityError, UnexpectedActivityTypeError from utils import activitypub_utils from config import USERNAME, BASE_URL, ID from config import CTX_AS, CTX_SECURITY, AS_PUBLIC @@ -24,11 +25,13 @@ from typing import TypeVar logger = logging.getLogger(__name__) +# Helper/shortcut for typing ObjectType = Dict[str, Any] ObjectOrIDType = Union[str, ObjectType] class ActivityType(Enum): + """Supported activity `type`.""" ANNOUNCE = 'Announce' BLOCK = 'Block' LIKE = 'Like' @@ -50,10 +53,12 @@ class ActivityType(Enum): def random_object_id() -> str: + """Generates a random object ID.""" return binascii.hexlify(os.urandom(8)).decode('utf-8') def _remove_id(doc: ObjectType) -> ObjectType: + """Helper for removing MongoDB's `_id` field.""" doc = doc.copy() if '_id' in doc: del(doc['_id']) @@ -61,13 +66,16 @@ def _remove_id(doc: ObjectType) -> ObjectType: def _to_list(data: Union[List[Any], Any]) -> List[Any]: + """Helper to convert fields that can be either an object or a list of objects to a list of object.""" if isinstance(data, list): return data return [data] def clean_activity(activity: ObjectType) -> Dict[str, Any]: - # Remove the hidden bco and bcc field + """Clean the activity before rendering it. + - Remove the hidden bco and bcc field + """ for field in ['bto', 'bcc']: if field in activity: del(activity[field]) @@ -77,23 +85,30 @@ def clean_activity(activity: ObjectType) -> Dict[str, Any]: def _get_actor_id(actor: ObjectOrIDType) -> str: + """Helper for retrieving an actor `id`.""" if isinstance(actor, dict): return actor['id'] return actor class BaseActivity(object): + """Base class for ActivityPub activities.""" + ACTIVITY_TYPE: Optional[ActivityType] = None ALLOWED_OBJECT_TYPES: List[ActivityType] = [] def __init__(self, **kwargs) -> None: + # Ensure the class has an activity type defined if not self.ACTIVITY_TYPE: - raise ValueError('Missing ACTIVITY_TYPE') + raise BadActivityError('Missing ACTIVITY_TYPE') + # Ensure the activity has a type and a valid one if kwargs.get('type') is not None and kwargs.pop('type') != self.ACTIVITY_TYPE.value: - raise ValueError('Expect the type to be {}'.format(self.ACTIVITY_TYPE)) + raise UnexpectedActivityTypeError('Expect the type to be {}'.format(self.ACTIVITY_TYPE)) + # Initialize the object self._data: Dict[str, Any] = {'type': self.ACTIVITY_TYPE.value} + logger.debug(f'initializing a {self.ACTIVITY_TYPE.value} activity') if 'id' in kwargs: self._data['id'] = kwargs.pop('id') @@ -115,12 +130,13 @@ class BaseActivity(object): self._data['object'] = obj else: if not self.ALLOWED_OBJECT_TYPES: - raise ValueError('unexpected object') + raise UnexpectedActivityTypeError('unexpected object') if 'type' not in obj or (self.ACTIVITY_TYPE != ActivityType.CREATE and 'id' not in obj): - raise ValueError('invalid object') + raise BadActivityError('invalid object, missing type') if ActivityType(obj['type']) not in self.ALLOWED_OBJECT_TYPES: - print(self, kwargs) - raise ValueError(f'unexpected object type {obj["type"]} (allowed={self.ALLOWED_OBJECT_TYPES})') + raise UnexpectedActivityTypeError( + f'unexpected object type {obj["type"]} (allowed={self.ALLOWED_OBJECT_TYPES})' + ) self._data['object'] = obj if '@context' not in kwargs: @@ -144,6 +160,7 @@ class BaseActivity(object): allowed_keys = None try: allowed_keys = self._init(**kwargs) + logger.debug('calling custom init') except NotImplementedError: pass @@ -151,7 +168,7 @@ class BaseActivity(object): # Allows an extra to (like for Accept and Follow) kwargs.pop('to', None) if len(set(kwargs.keys()) - set(allowed_keys)) > 0: - raise ValueError('extra data left: {}'.format(kwargs)) + raise BadActivityError('extra data left: {}'.format(kwargs)) else: # Remove keys with `None` value valid_kwargs = {} @@ -177,7 +194,7 @@ class BaseActivity(object): return '{}({!r})'.format(self.__class__.__qualname__, self._data.get('id')) def __str__(self) -> str: - return str(self._data['id']) + return str(self._data.get('id', f'[new {self.ACTIVITY_TYPE} activity]')) def __getattr__(self, name: str) -> Any: if self._data.get(name): @@ -191,6 +208,7 @@ class BaseActivity(object): raise NotImplementedError def set_id(self, uri: str, obj_id: str) -> None: + logger.debug(f'setting ID {uri} / {obj_id}') self._data['id'] = uri try: self._set_id(uri, obj_id) @@ -227,7 +245,7 @@ class BaseActivity(object): else: obj = OBJECT_SERVICE.get(self._data['object']) if ActivityType(obj.get('type')) not in self.ALLOWED_OBJECT_TYPES: - raise ValueError('invalid object type') + raise UnexpectedActivityTypeError(f'invalid object type {obj.get("type")}') p = parse_activity(obj) @@ -275,18 +293,19 @@ class BaseActivity(object): raise NotImplementedError def process_from_inbox(self) -> None: + logger.debug(f'calling main process from inbox hook for {self}') self.verify() actor = self.get_actor() if DB.outbox.find_one({'type': ActivityType.BLOCK.value, 'activity.object': actor.id, 'meta.undo': False}): - print('actor is blocked, drop activity') + logger.info(f'actor {actor} is blocked, dropping the received activity {self}') return if DB.inbox.find_one({'remote_id': self.id}): # The activity is already in the inbox - print('received duplicate activity') + logger.info(f'received duplicate activity {self}, dropping it') return activity = self.to_dict() @@ -296,13 +315,16 @@ class BaseActivity(object): 'remote_id': self.id, 'meta': {'undo': False, 'deleted': False}, }) + logger.info('activity {self} saved') try: self._process_from_inbox() + logger.debug('called process from inbox hook') except NotImplementedError: - pass + logger.debug('process from inbox hook not implemented') def post_to_outbox(self) -> None: + logger.debug(f'calling main post to outbox hook for {self}') obj_id = random_object_id() self.set_id(f'{ID}/outbox/{obj_id}', obj_id) self.verify() @@ -316,19 +338,20 @@ class BaseActivity(object): }) recipients = self.recipients() + logger.info(f'recipients={recipients}') activity = clean_activity(activity) try: self._post_to_outbox(obj_id, activity, recipients) + logger.debug(f'called post to outbox hook') except NotImplementedError: - pass + logger.debug('post to outbox hook not implemented') generate_signature(activity, KEY.privkey) payload = json.dumps(activity) - print('will post') for recp in recipients: + logger.debug(f'posting to {recp}') self._post_to_inbox(payload, recp) - print('done') def _post_to_inbox(self, payload: str, to: str): tasks.post_to_inbox.delay(payload, to) @@ -371,8 +394,8 @@ class BaseActivity(object): if shared_inbox not in out: out.append(shared_inbox) continue - if col_actor.inbox and col_actor.inbox not in out: - out.append(col_actor.inbox) + if col_actor.inbox and col_actor.inbox not in out: + out.append(col_actor.inbox) continue diff --git a/app.py b/app.py index 21a61a2..6b11aa0 100644 --- a/app.py +++ b/app.py @@ -810,10 +810,11 @@ def api_new_note(): return Response( status=201, response='OK', - headers={'Microblogpub-Created-Activity': created.id}, + headers={'Microblogpub-Created-Activity': create.id}, ) @app.route('/api/stream') +@api_required def api_stream(): return Response( response=json.dumps(activitypub.build_inbox_json_feed('/api/stream', request.args.get('cursor'))), diff --git a/tests/federation_test.py b/tests/federation_test.py index ff8db17..3bd1021 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -18,6 +18,7 @@ class Instance(object): self.host_url = host_url self.docker_url = docker_url or host_url self.session = requests.Session() + self._create_delay = 8 def _do_req(self, url, headers): url = url.replace(self.docker_url, self.host_url) @@ -57,7 +58,14 @@ class Instance(object): assert resp.status_code == 201 # We need to wait for the Follow/Accept dance - time.sleep(10) + time.sleep(self._create_delay) + return resp.headers.get('microblogpub-created-activity') + + def new_note(self, content): + resp = self.session.get(f'{self.host_url}/api/new_note', params={'content': content}) + assert resp.status_code == 201 + + time.sleep(self._create_delay) return resp.headers.get('microblogpub-created-activity') def undo(self, oid: str) -> None: @@ -65,7 +73,7 @@ class Instance(object): assert resp.status_code == 201 # We need to wait for the Follow/Accept dance - time.sleep(10) + time.sleep(self._create_delay) return resp.headers.get('microblogpub-created-activity') def followers(self): @@ -89,6 +97,11 @@ class Instance(object): resp.raise_for_status() return resp.json() + def stream_jsonfeed(self): + resp = self.session.get(f'{self.host_url}/api/stream', headers={'Accept': 'application/json'}) + resp.raise_for_status() + return resp.json() + def _instances(): instance1 = Instance('http://localhost:5006', 'http://instance1_web_1:5005') @@ -111,12 +124,10 @@ def test_follow(): # Instance1 follows instance2 instance1.follow(instance2) instance1_debug = instance1.debug() - print(f'instance1_debug={instance1_debug}') assert instance1_debug['inbox'] == 1 # An Accept activity should be there assert instance1_debug['outbox'] == 1 # We've sent a Follow activity instance2_debug = instance2.debug() - print(f'instance2_debug={instance2_debug}') assert instance2_debug['inbox'] == 1 # An Follow activity should be there assert instance2_debug['outbox'] == 1 # We've sent a Accept activity @@ -152,4 +163,21 @@ def test_follow_unfollow(): assert instance2_debug['inbox'] == 2 # An Follow and Undo activity should be there assert instance2_debug['outbox'] == 1 # We've sent a Accept activity +def test_post_content(): + instance1, instance2 = _instances() + # Instance1 follows instance2 + instance1.follow(instance2) + instance2.follow(instance1) + inbox_stream = instance2.stream_jsonfeed() + assert len(inbox_stream['items']) == 0 + + create_id = instance1.new_note('hello') + instance2_debug = instance2.debug() + assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there + instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + + # Ensure the post is visible in instance2's stream + inbox_stream = instance2.stream_jsonfeed() + assert len(inbox_stream['items']) == 1 + assert inbox_stream['items'][0]['id'] == create_id From 443378a057beadf4c9de2fa28b6419e926c00fd7 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 27 May 2018 22:43:46 +0200 Subject: [PATCH 0052/1425] Tweak the test setting --- tests/federation_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/federation_test.py b/tests/federation_test.py index 3bd1021..971b43c 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -18,7 +18,7 @@ class Instance(object): self.host_url = host_url self.docker_url = docker_url or host_url self.session = requests.Session() - self._create_delay = 8 + self._create_delay = 10 def _do_req(self, url, headers): url = url.replace(self.docker_url, self.host_url) From ff95e6773e9dd81d3862b87cbacb73cdd03b61b5 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 27 May 2018 22:47:33 +0200 Subject: [PATCH 0053/1425] Fix flake8 warnings --- activitypub.py | 3 +-- docker-compose-dev.yml | 2 +- docker-compose-tests.yml | 1 + docker-compose.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/activitypub.py b/activitypub.py index 95ce35a..3ce0441 100644 --- a/activitypub.py +++ b/activitypub.py @@ -21,7 +21,6 @@ from config import PUBLIC_INSTANCES import tasks from typing import List, Optional, Dict, Any, Union -from typing import TypeVar logger = logging.getLogger(__name__) @@ -345,7 +344,7 @@ class BaseActivity(object): self._post_to_outbox(obj_id, activity, recipients) logger.debug(f'called post to outbox hook') except NotImplementedError: - logger.debug('post to outbox hook not implemented') + logger.debug('post to outbox hook not implemented') generate_signature(activity, KEY.privkey) payload = json.dumps(activity) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 0967722..bf6b0ee 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -1,4 +1,4 @@ -version: '3' +version: '2' services: celery: build: . diff --git a/docker-compose-tests.yml b/docker-compose-tests.yml index d45609b..06f3f05 100644 --- a/docker-compose-tests.yml +++ b/docker-compose-tests.yml @@ -15,6 +15,7 @@ services: - MICROBLOGPUB_MONGODB_HOST=mongo:27017 - MICROBLOGPUB_DEBUG=1 celery: +# image: "instance1_web" build: . links: - mongo diff --git a/docker-compose.yml b/docker-compose.yml index c2dab44..74f92d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: '2' services: web: build: . From 9f9f79edb56814e1e2ecc2e88bee5f4f21ddca9c Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 28 May 2018 19:46:23 +0200 Subject: [PATCH 0054/1425] Tests tests tests and bugfixes --- .dockerignore | 1 + activitypub.py | 66 ++++++++++++------ app.py | 72 +++++++++++++++----- docker-compose-tests.yml | 2 + docker-compose.yml | 4 +- tasks.py | 3 +- tests/federation_test.py | 142 +++++++++++++++++++++++++++++++++++++++ utils/httpsig.py | 6 ++ 8 files changed, 258 insertions(+), 38 deletions(-) diff --git a/.dockerignore b/.dockerignore index 35249de..ccc0367 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,3 @@ +__pycache__/ data/ tests/ diff --git a/activitypub.py b/activitypub.py index 3ce0441..252feca 100644 --- a/activitypub.py +++ b/activitypub.py @@ -251,16 +251,19 @@ class BaseActivity(object): self.__obj: BaseActivity = p return p - def _to_dict(self, data: ObjectType) -> ObjectType: - return data - - def to_dict(self, embed: bool = False) -> ObjectType: + def to_dict(self, embed: bool = False, embed_object_id_only: bool = False) -> ObjectType: data = dict(self._data) if embed: for k in ['@context', 'signature']: if k in data: del(data[k]) - return self._to_dict(data) + if data.get('object') and embed_object_id_only and isinstance(data['object'], dict): + try: + data['object'] = data['object']['id'] + except KeyError: + raise BadActivityError('embedded object does not have an id') + + return data def get_actor(self) -> 'BaseActivity': actor = self._data.get('actor') @@ -424,11 +427,6 @@ class Person(BaseActivity): def _verify(self) -> None: ACTOR_SERVICE.get(self._data['id']) - def _to_dict(self, data): - # if 'icon' in data: - # data['icon'] = data['icon'].to_dict() - return data - class Block(BaseActivity): ACTIVITY_TYPE = ActivityType.BLOCK @@ -568,12 +566,19 @@ class Like(BaseActivity): def _process_from_inbox(self): obj = self.get_object() # Update the meta counter if the object is published by the server - DB.outbox.update_one({'activity.object.id': obj.id}, {'$inc': {'meta.count_like': 1}}) + DB.outbox.update_one({'activity.object.id': obj.id}, { + '$inc': {'meta.count_like': 1}, + '$addToSet': {'meta.col_likes': self.to_dict(embed=True, embed_object_id_only=True)}, + }) + # XXX(tsileo): notification?? def _undo_inbox(self) -> None: obj = self.get_object() # Update the meta counter if the object is published by the server - DB.outbox.update_one({'activity.object.id': obj.id}, {'$inc': {'meta.count_like': -1}}) + DB.outbox.update_one({'activity.object.id': obj.id}, { + '$inc': {'meta.count_like': -1}, + '$pull': {'meta.col_likes': {'id': self.id}}, + }) def _undo_should_purge_cache(self) -> bool: # If a like coutn was decremented, we need to purge the application cache @@ -582,19 +587,26 @@ class Like(BaseActivity): def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]): obj = self.get_object() # Unlikely, but an actor can like it's own post - DB.outbox.update_one({'activity.object.id': obj.id}, {'$inc': {'meta.count_like': 1}}) + DB.outbox.update_one({'activity.object.id': obj.id}, { + '$inc': {'meta.count_like': 1}, + '$addToSet': {'meta.col_likes': self.to_dict(embed=True, embed_object_id_only=True)}, + }) + # Keep track of the like we just performed DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.liked': obj_id}}) def _undo_outbox(self) -> None: obj = self.get_object() # Unlikely, but an actor can like it's own post - DB.outbox.update_one({'activity.object.id': obj.id}, {'$inc': {'meta.count_like': -1}}) + DB.outbox.update_one({'activity.object.id': obj.id}, { + '$inc': {'meta.count_like': -1}, + '$pull': {'meta.col_likes': {'id': self.id}}, + }) DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.liked': False}}) def build_undo(self) -> BaseActivity: - return Undo(object=self.to_dict(embed=True)) + return Undo(object=self.to_dict(embed=True, embed_object_id_only=True)) class Announce(BaseActivity): @@ -613,7 +625,9 @@ class Announce(BaseActivity): def _process_from_inbox(self) -> None: if isinstance(self._data['object'], str) and not self._data['object'].startswith('http'): # TODO(tsileo): actually drop it without storing it and better logging, also move the check somewhere else - print(f'received an Annouce referencing an OStatus notice ({self._data["object"]}), dropping the message') + logger.warn( + f'received an Annouce referencing an OStatus notice ({self._data["object"]}), dropping the message' + ) return # Save/cache the object, and make it part of the stream so we can fetch it if isinstance(self._data['object'], str): @@ -626,12 +640,18 @@ class Announce(BaseActivity): obj = parse_activity(raw_obj) else: obj = self.get_object() - DB.outbox.update_one({'activity.object.id': obj.id}, {'$inc': {'meta.count_boost': 1}}) + DB.outbox.update_one({'activity.object.id': obj.id}, { + '$inc': {'meta.count_boost': 1}, + '$addToSet': {'meta.col_shares': self.to_dict(embed=True, embed_object_id_only=True)}, + }) def _undo_inbox(self) -> None: obj = self.get_object() - DB.inbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) - DB.outbox.update_one({'activity.object.id': obj.id}, {'$inc': {'meta.count_boost': -1}}) + # Update the meta counter if the object is published by the server + DB.outbox.update_one({'activity.object.id': obj.id}, { + '$inc': {'meta.count_boost': -1}, + '$pull': {'meta.col_shares': {'id': self.id}}, + }) def _undo_should_purge_cache(self) -> bool: # If a like coutn was decremented, we need to purge the application cache @@ -971,6 +991,14 @@ def parse_collection(payload: Optional[Dict[str, Any]] = None, url: Optional[str return activitypub_utils.parse_collection(payload, url) +def embed_collection(data): + return { + "type": "Collection", + "totalItems": len(data), + "items": data, + } + + def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, col_name=None): col_name = col_name or col.name if q is None: diff --git a/app.py b/app.py index 6b11aa0..a5f47e2 100644 --- a/app.py +++ b/app.py @@ -35,6 +35,7 @@ import activitypub import config from activitypub import ActivityType from activitypub import clean_activity +from activitypub import embed_collection from utils.content_helper import parse_markdown from config import KEY from config import DB @@ -56,10 +57,13 @@ from utils.key import get_secret_key from utils.webfinger import get_remote_follow_template from utils.webfinger import get_actor_url +from typing import Dict, Any app = Flask(__name__) app.secret_key = get_secret_key('flask') +logger = logging.getLogger(__name__) + # Hook up Flask logging with gunicorn gunicorn_logger = logging.getLogger('gunicorn.error') root_logger = logging.getLogger() @@ -435,6 +439,25 @@ def webfinger(): headers={'Content-Type': 'application/jrd+json; charset=utf-8' if not app.debug else 'application/json'}, ) + +def add_extra_collection(raw_doc: Dict[str, Any]) -> Dict[str, Any]: + if 'col_likes' in raw_doc.get('meta', {}): + col_likes = raw_doc['meta']['col_likes'] + if raw_doc['activity']['type'] == ActivityType.CREATE.value: + raw_doc['activity']['object']['likes'] = embed_collection(col_likes) + if 'col_shares' in raw_doc.get('meta', {}): + col_shares = raw_doc['meta']['col_shares'] + if raw_doc['activity']['type'] == ActivityType.CREATE.value: + raw_doc['activity']['object']['shares'] = embed_collection(col_shares) + + return raw_doc + + +def activity_from_doc(raw_doc: Dict[str, Any]) -> Dict[str, Any]: + raw_doc = add_extra_collection(raw_doc) + return clean_activity(raw_doc['activity']) + + @app.route('/outbox', methods=['GET', 'POST']) def outbox(): if request.method == 'GET': @@ -444,7 +467,7 @@ def outbox(): # FIXME(tsileo): filter deleted, add query support for build_ordered_collection q = { 'meta.deleted': False, - 'type': {'$in': [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, + #'type': {'$in': [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, } return jsonify(**activitypub.build_ordered_collection( DB.outbox, @@ -477,7 +500,7 @@ def outbox(): @app.route('/outbox/') def outbox_detail(item_id): doc = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) - return jsonify(**clean_activity(doc['activity'])) + return jsonify(**activity_from_doc(doc)) @app.route('/outbox//activity') @@ -485,10 +508,11 @@ def outbox_activity(item_id): data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) if not data: abort(404) - obj = data['activity'] + obj = activity_from_doc(data) if obj['type'] != ActivityType.CREATE.value: abort(404) - return jsonify(**clean_activity(obj['object'])) + return jsonify(**obj['object']) + @app.route('/admin', methods=['GET']) @login_required @@ -597,23 +621,38 @@ def notifications(): cursor=cursor, ) -@app.route('/ui/boost') -@login_required -def ui_boost(): + +@app.route('/api/boost') +@api_required +def api_boost(): oid = request.args.get('id') obj = activitypub.parse_activity(OBJECT_SERVICE.get(oid)) announce = obj.build_announce() announce.post_to_outbox() - return redirect(request.args.get('redirect')) + if request.args.get('redirect'): + return redirect(request.args.get('redirect')) + return Response( + status=201, + headers={'Microblogpub-Created-Activity': announce.id}, + ) -@app.route('/ui/like') -@login_required -def ui_like(): +@app.route('/api/like') +@api_required +def api_like(): + # FIXME(tsileo): ensure a Note and not a Create is given oid = request.args.get('id') obj = activitypub.parse_activity(OBJECT_SERVICE.get(oid)) + if not obj: + raise ValueError(f'unkown {oid} object') like = obj.build_like() like.post_to_outbox() - return redirect(request.args.get('redirect')) + if request.args.get('redirect'): + return redirect(request.args.get('redirect')) + return Response( + status=201, + headers={'Microblogpub-Created-Activity': like.id}, + ) + @app.route('/api/undo', methods=['GET', 'POST']) @api_required @@ -702,19 +741,20 @@ def inbox(): )) data = request.get_json(force=True) - print(data) + logger.debug(f'req_headers={request.headers}') + logger.debug(f'raw_data={data}') try: print(verify_request(ACTOR_SERVICE)) except Exception: - print('failed to verify request, trying to verify the payload by fetching the remote') + logger.exception('failed to verify request, trying to verify the payload by fetching the remote') try: data = OBJECT_SERVICE.get(data['id']) except Exception: - print(f'failed to fetch remote id at {data["id"]}') + logger.exception(f'failed to fetch remote id at {data["id"]}') abort(422) activity = activitypub.parse_activity(data) - print(activity) + logger.debug(f'inbox activity={activity}/{data}') activity.process_from_inbox() return Response( diff --git a/docker-compose-tests.yml b/docker-compose-tests.yml index 06f3f05..a834b93 100644 --- a/docker-compose-tests.yml +++ b/docker-compose-tests.yml @@ -21,6 +21,8 @@ services: - mongo - rmq command: 'celery worker -l info -A tasks' + volumes: + - "${CONFIG_DIR}:/app/config" environment: - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// - MICROBLOGPUB_MONGODB_HOST=mongo:27017 diff --git a/docker-compose.yml b/docker-compose.yml index 74f92d8..c7f07b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,9 @@ services: - mongo - rmq command: 'celery worker -l info -A tasks' - environment: + volumes: + - "${CONFIG_DIR}:/app/config" + environment: - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// - MICROBLOGPUB_MONGODB_HOST=mongo:27017 mongo: diff --git a/tasks.py b/tasks.py index 7079491..7e45581 100644 --- a/tasks.py +++ b/tasks.py @@ -15,7 +15,7 @@ from utils.httpsig import HTTPSigAuth from utils.opengraph import fetch_og_metadata -log = logging.getLogger() +log = logging.getLogger(__name__) app = Celery('tasks', broker=os.getenv('MICROBLOGPUB_AMQP_BROKER', 'pyamqp://guest@localhost//')) # app = Celery('tasks', broker='pyamqp://guest@rabbitmq//') SigAuth = HTTPSigAuth(ID+'#main-key', KEY.privkey) @@ -31,7 +31,6 @@ def post_to_inbox(self, payload, to): 'Accept': HEADERS[1], 'User-Agent': USER_AGENT, }) - print(resp) log.info('resp=%s', resp) log.info('resp_body=%s', resp.text) resp.raise_for_status() diff --git a/tests/federation_test.py b/tests/federation_test.py index 971b43c..0b3c2e9 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -68,6 +68,20 @@ class Instance(object): time.sleep(self._create_delay) return resp.headers.get('microblogpub-created-activity') + def boost(self, activity_id): + resp = self.session.get(f'{self.host_url}/api/boost', params={'id': activity_id}) + assert resp.status_code == 201 + + time.sleep(self._create_delay) + return resp.headers.get('microblogpub-created-activity') + + def like(self, activity_id): + resp = self.session.get(f'{self.host_url}/api/like', params={'id': activity_id}) + assert resp.status_code == 201 + + time.sleep(self._create_delay) + return resp.headers.get('microblogpub-created-activity') + def undo(self, oid: str) -> None: resp = self.session.get(f'{self.host_url}/api/undo', params={'id': oid}) assert resp.status_code == 201 @@ -97,6 +111,11 @@ class Instance(object): resp.raise_for_status() return resp.json() + def outbox_get(self, aid): + resp = self.session.get(aid.replace(self.docker_url, self.host_url), headers={'Accept': 'application/activity+json'}) + resp.raise_for_status() + return resp.json() + def stream_jsonfeed(self): resp = self.session.get(f'{self.host_url}/api/stream', headers={'Accept': 'application/json'}) resp.raise_for_status() @@ -163,6 +182,7 @@ def test_follow_unfollow(): assert instance2_debug['inbox'] == 2 # An Follow and Undo activity should be there assert instance2_debug['outbox'] == 1 # We've sent a Accept activity + def test_post_content(): instance1, instance2 = _instances() # Instance1 follows instance2 @@ -181,3 +201,125 @@ def test_post_content(): inbox_stream = instance2.stream_jsonfeed() assert len(inbox_stream['items']) == 1 assert inbox_stream['items'][0]['id'] == create_id + + +def test_post_content_and_like(): + instance1, instance2 = _instances() + # Instance1 follows instance2 + instance1.follow(instance2) + instance2.follow(instance1) + + create_id = instance1.new_note('hello') + + # Ensure the post is visible in instance2's stream + inbox_stream = instance2.stream_jsonfeed() + assert len(inbox_stream['items']) == 1 + assert inbox_stream['items'][0]['id'] == create_id + + # Now, instance2 like the note + like_id = instance2.like(f'{create_id}/activity') + + instance1_debug = instance1.debug() + assert instance1_debug['inbox'] == 3 # Follow, Accept and Like + assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + + note = instance1.outbox_get(f'{create_id}/activity') + assert 'likes' in note + assert len(note['likes']['items']) == 1 + assert note['likes']['items'][0]['id'] == like_id + +def test_post_content_and_like_unlike(): + instance1, instance2 = _instances() + # Instance1 follows instance2 + instance1.follow(instance2) + instance2.follow(instance1) + + create_id = instance1.new_note('hello') + + # Ensure the post is visible in instance2's stream + inbox_stream = instance2.stream_jsonfeed() + assert len(inbox_stream['items']) == 1 + assert inbox_stream['items'][0]['id'] == create_id + + # Now, instance2 like the note + like_id = instance2.like(f'{create_id}/activity') + + instance1_debug = instance1.debug() + assert instance1_debug['inbox'] == 3 # Follow, Accept and Like + assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + + note = instance1.outbox_get(f'{create_id}/activity') + assert 'likes' in note + assert len(note['likes']['items']) == 1 + assert note['likes']['items'][0]['id'] == like_id + + instance2.undo(like_id) + + instance1_debug = instance1.debug() + assert instance1_debug['inbox'] == 4 # Follow, Accept and Like and Undo + assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + + note = instance1.outbox_get(f'{create_id}/activity') + assert 'likes' in note + assert len(note['likes']['items']) == 0 + +def test_post_content_and_boost(): + instance1, instance2 = _instances() + # Instance1 follows instance2 + instance1.follow(instance2) + instance2.follow(instance1) + + create_id = instance1.new_note('hello') + + # Ensure the post is visible in instance2's stream + inbox_stream = instance2.stream_jsonfeed() + assert len(inbox_stream['items']) == 1 + assert inbox_stream['items'][0]['id'] == create_id + + # Now, instance2 like the note + boost_id = instance2.boost(f'{create_id}/activity') + + instance1_debug = instance1.debug() + assert instance1_debug['inbox'] == 3 # Follow, Accept and Announce + assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + + note = instance1.outbox_get(f'{create_id}/activity') + assert 'shares' in note + assert len(note['shares']['items']) == 1 + assert note['shares']['items'][0]['id'] == boost_id + + +def test_post_content_and_boost_unboost(): + instance1, instance2 = _instances() + # Instance1 follows instance2 + instance1.follow(instance2) + instance2.follow(instance1) + + create_id = instance1.new_note('hello') + + # Ensure the post is visible in instance2's stream + inbox_stream = instance2.stream_jsonfeed() + assert len(inbox_stream['items']) == 1 + assert inbox_stream['items'][0]['id'] == create_id + + # Now, instance2 like the note + boost_id = instance2.boost(f'{create_id}/activity') + + instance1_debug = instance1.debug() + assert instance1_debug['inbox'] == 3 # Follow, Accept and Announce + assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + + note = instance1.outbox_get(f'{create_id}/activity') + assert 'shares' in note + assert len(note['shares']['items']) == 1 + assert note['shares']['items'][0]['id'] == boost_id + + instance2.undo(boost_id) + + instance1_debug = instance1.debug() + assert instance1_debug['inbox'] == 4 # Follow, Accept and Announce and Undo + assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + + note = instance1.outbox_get(f'{create_id}/activity') + assert 'shares' in note + assert len(note['shares']['items']) == 0 diff --git a/utils/httpsig.py b/utils/httpsig.py index 84245fb..f63c12c 100644 --- a/utils/httpsig.py +++ b/utils/httpsig.py @@ -8,6 +8,7 @@ from urllib.parse import urlparse from typing import Any, Dict import base64 import hashlib +import logging from flask import request from requests.auth import AuthBase @@ -15,6 +16,8 @@ from requests.auth import AuthBase from Crypto.Signature import PKCS1_v1_5 from Crypto.Hash import SHA256 +logger = logging.getLogger(__name__) + def _build_signed_string(signed_headers: str, method: str, path: str, headers: Any, body_digest: str) -> str: out = [] @@ -51,6 +54,7 @@ def _body_digest() -> str: def verify_request(actor_service) -> bool: hsig = _parse_sig_header(request.headers.get('Signature')) + logger.debug(f'hsig={hsig}') signed_string = _build_signed_string(hsig['headers'], request.method, request.path, request.headers, _body_digest()) _, rk = actor_service.get_public_key(hsig['keyId']) return _verify_h(signed_string, base64.b64decode(hsig['signature']), rk) @@ -62,6 +66,7 @@ class HTTPSigAuth(AuthBase): self.privkey = privkey def __call__(self, r): + logger.info(f'keyid={self.keyid}') host = urlparse(r.url).netloc bh = hashlib.new('sha256') bh.update(r.body.encode('utf-8')) @@ -79,5 +84,6 @@ class HTTPSigAuth(AuthBase): headers = { 'Signature': f'keyId="{self.keyid}",algorithm="rsa-sha256",headers="{sigheaders}",signature="{sig}"' } + logger.info(f'signed request headers={headers}') r.headers.update(headers) return r From 64c1496c570b9bc22c65d0da5664b8363c2e1c2e Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 28 May 2018 19:48:30 +0200 Subject: [PATCH 0055/1425] Fix docker-compose file --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index c7f07b5..0218494 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ services: command: 'celery worker -l info -A tasks' volumes: - "${CONFIG_DIR}:/app/config" - environment: + environment: - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// - MICROBLOGPUB_MONGODB_HOST=mongo:27017 mongo: From bba598be66e5cf031e87fb187742c7629d1731b7 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 28 May 2018 22:38:48 +0200 Subject: [PATCH 0056/1425] Add federation test case for note deletion --- activitypub.py | 21 ++++++++++++++++----- app.py | 16 ++++++++++++++++ tests/federation_test.py | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 5 deletions(-) diff --git a/activitypub.py b/activitypub.py index 252feca..7610a69 100644 --- a/activitypub.py +++ b/activitypub.py @@ -119,7 +119,7 @@ class BaseActivity(object): actor = self._validate_person(actor) self._data['actor'] = actor else: - if not self.NO_CONTEXT: + if not self.NO_CONTEXT and self.ACTIVITY_TYPE != ActivityType.TOMBSTONE: actor = ID self._data['actor'] = actor @@ -299,6 +299,7 @@ class BaseActivity(object): self.verify() actor = self.get_actor() + # Check for Block activity if DB.outbox.find_one({'type': ActivityType.BLOCK.value, 'activity.object': actor.id, 'meta.undo': False}): @@ -415,6 +416,9 @@ class BaseActivity(object): def build_undo(self) -> 'BaseActivity': raise NotImplementedError + def build_delete(self) -> 'BaseActivity': + raise NotImplementedError + class Person(BaseActivity): ACTIVITY_TYPE = ActivityType.PERSON @@ -645,6 +649,7 @@ class Announce(BaseActivity): '$inc': {'meta.count_boost': 1}, '$addToSet': {'meta.col_shares': self.to_dict(embed=True, embed_object_id_only=True)}, }) + def _undo_inbox(self) -> None: obj = self.get_object() # Update the meta counter if the object is published by the server @@ -683,14 +688,17 @@ class Delete(BaseActivity): ALLOWED_OBJECT_TYPES = [ActivityType.NOTE, ActivityType.TOMBSTONE] def _recipients(self) -> List[str]: - return self.get_object().recipients() + obj = self.get_object() + if obj.type_enum == ActivityType.TOMBSTONE: + obj = parse_activity(OBJECT_SERVICE.get(obj.id)) + return obj._recipients() - def _process_from_inbox(self): + def _process_from_inbox(self) -> None: DB.inbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) # TODO(tsileo): also delete copies stored in parents' `meta.replies` # TODO(tsileo): also purge the cache if it's a reply of a published activity - def _post_to_outbox(self, obj_id, activity, recipients): + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: DB.outbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) @@ -870,6 +878,9 @@ class Note(BaseActivity): published=datetime.utcnow().replace(microsecond=0).isoformat() + 'Z', ) + def build_delete(self) -> BaseActivity: + return Delete(object=Tombstone(id=self.id).to_dict(embed=True)) + _ACTIVITY_TYPE_TO_CLS = { ActivityType.IMAGE: Image, @@ -946,7 +957,7 @@ def build_inbox_json_feed(path: str, request_cursor: Optional[str] = None) -> Di data = [] cursor = None - q: Dict[str, Any] = {'type': 'Create'} + q: Dict[str, Any] = {'type': 'Create', 'meta.deleted': False} if request_cursor: q['_id'] = {'$lt': request_cursor} diff --git a/app.py b/app.py index a5f47e2..b28715b 100644 --- a/app.py +++ b/app.py @@ -622,9 +622,25 @@ def notifications(): ) +@app.route('/api/delete') +@api_required +def api_delete(): + # FIXME(tsileo): ensure a Note and not a Create is given + oid = request.args.get('id') + obj = activitypub.parse_activity(OBJECT_SERVICE.get(oid)) + delete = obj.build_delete() + delete.post_to_outbox() + if request.args.get('redirect'): + return redirect(request.args.get('redirect')) + return Response( + status=201, + headers={'Microblogpub-Created-Activity': delete.id}, + ) + @app.route('/api/boost') @api_required def api_boost(): + # FIXME(tsileo): ensure a Note and not a Create is given oid = request.args.get('id') obj = activitypub.parse_activity(OBJECT_SERVICE.get(oid)) announce = obj.build_announce() diff --git a/tests/federation_test.py b/tests/federation_test.py index 0b3c2e9..9da231d 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -82,6 +82,13 @@ class Instance(object): time.sleep(self._create_delay) return resp.headers.get('microblogpub-created-activity') + def delete(self, oid: str) -> None: + resp = self.session.get(f'{self.host_url}/api/delete', params={'id': oid}) + assert resp.status_code == 201 + + time.sleep(self._create_delay) + return resp.headers.get('microblogpub-created-activity') + def undo(self, oid: str) -> None: resp = self.session.get(f'{self.host_url}/api/undo', params={'id': oid}) assert resp.status_code == 201 @@ -203,6 +210,35 @@ def test_post_content(): assert inbox_stream['items'][0]['id'] == create_id +def test_post_content_and_delete(): + instance1, instance2 = _instances() + # Instance1 follows instance2 + instance1.follow(instance2) + instance2.follow(instance1) + + inbox_stream = instance2.stream_jsonfeed() + assert len(inbox_stream['items']) == 0 + + create_id = instance1.new_note('hello') + instance2_debug = instance2.debug() + assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there + instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + + # Ensure the post is visible in instance2's stream + inbox_stream = instance2.stream_jsonfeed() + assert len(inbox_stream['items']) == 1 + assert inbox_stream['items'][0]['id'] == create_id + + instance1.delete(f'{create_id}/activity') + instance2_debug = instance2.debug() + assert instance2_debug['inbox'] == 4 # An Follow, Accept and Create and Delete activity should be there + instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + + # Ensure the post has been delete from instance2's stream + inbox_stream = instance2.stream_jsonfeed() + assert len(inbox_stream['items']) == 0 + + def test_post_content_and_like(): instance1, instance2 = _instances() # Instance1 follows instance2 From b727c8c9c71b25bd670e00238894375b94907a1d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 29 May 2018 00:12:44 +0200 Subject: [PATCH 0057/1425] Add tests for block, and start test for replies --- app.py | 32 +++++++++++++++- tests/federation_test.py | 82 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 108 insertions(+), 6 deletions(-) diff --git a/app.py b/app.py index b28715b..9d86289 100644 --- a/app.py +++ b/app.py @@ -847,9 +847,16 @@ def api_new_note(): source = request.args.get('content') if not source: raise ValueError('missing content') + + reply = None + if request.args.get('reply'): + reply = activitypub.parse_activity(OBJECT_SERVICE.get(request.args.get('reply'))) + source = request.args.get('content') content, tags = parse_markdown(source) to = request.args.get('to') cc = [ID+'/followers'] + if reply: + cc.append(reply.attributedTo) for tag in tags: if tag['type'] == 'Mention': cc.append(tag['href']) @@ -857,12 +864,14 @@ def api_new_note(): note = activitypub.Note( cc=cc, to=[to if to else config.AS_PUBLIC], - content=content, + content=content, # TODO(tsileo): handle markdown tag=tags, source={'mediaType': 'text/markdown', 'content': source}, + inReplyTo=reply.id if reply else None ) create = note.build_create() create.post_to_outbox() + return Response( status=201, response='OK', @@ -877,6 +886,27 @@ def api_stream(): headers={'Content-Type': 'application/json'}, ) + +@app.route('/api/block') +@api_required +def api_block(): + # FIXME(tsileo): ensure it's a Person ID + actor = request.args.get('actor') + if not actor: + raise ValueError('missing actor') + if DB.outbox.find_one({'type': ActivityType.BLOCK.value, + 'activity.object': actor, + 'meta.undo': False}): + return Response(status=201) + + block = activitypub.Block(object=actor) + block.post_to_outbox() + return Response( + status=201, + headers={'Microblogpub-Created-Activity': block.id}, + ) + + @app.route('/api/follow') @api_required def api_follow(): diff --git a/tests/federation_test.py b/tests/federation_test.py index 9da231d..a5bd1cf 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -52,6 +52,15 @@ class Instance(object): resp.raise_for_status() assert resp.status_code == 200 + def block(self, actor_url) -> None: + # Instance1 follows instance2 + resp = self.session.get(f'{self.host_url}/api/block', params={'actor': actor_url}) + assert resp.status_code == 201 + + # We need to wait for the Follow/Accept dance + time.sleep(self._create_delay/2) + return resp.headers.get('microblogpub-created-activity') + def follow(self, instance: 'Instance') -> None: # Instance1 follows instance2 resp = self.session.get(f'{self.host_url}/api/follow', params={'actor': instance.docker_url}) @@ -61,8 +70,11 @@ class Instance(object): time.sleep(self._create_delay) return resp.headers.get('microblogpub-created-activity') - def new_note(self, content): - resp = self.session.get(f'{self.host_url}/api/new_note', params={'content': content}) + def new_note(self, content, reply=None): + params = {'content': content} + if reply: + params['reply'] = reply + resp = self.session.get(f'{self.host_url}/api/new_note', params=params) assert resp.status_code == 201 time.sleep(self._create_delay) @@ -202,7 +214,7 @@ def test_post_content(): create_id = instance1.new_note('hello') instance2_debug = instance2.debug() assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity # Ensure the post is visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() @@ -210,6 +222,27 @@ def test_post_content(): assert inbox_stream['items'][0]['id'] == create_id +def test_block_and_post_content(): + instance1, instance2 = _instances() + # Instance1 follows instance2 + instance1.follow(instance2) + instance2.follow(instance1) + + inbox_stream = instance2.stream_jsonfeed() + assert len(inbox_stream['items']) == 0 + + instance2.block(instance1.docker_url) + + instance1.new_note('hello') + instance2_debug = instance2.debug() + assert instance2_debug['inbox'] == 2 # An Follow, Accept activity should be there, Create should have been dropped + assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow activity + the Block activity + + # Ensure the post is not visible in instance2's stream + inbox_stream = instance2.stream_jsonfeed() + assert len(inbox_stream['items']) == 0 + + def test_post_content_and_delete(): instance1, instance2 = _instances() # Instance1 follows instance2 @@ -222,7 +255,7 @@ def test_post_content_and_delete(): create_id = instance1.new_note('hello') instance2_debug = instance2.debug() assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity # Ensure the post is visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() @@ -232,7 +265,7 @@ def test_post_content_and_delete(): instance1.delete(f'{create_id}/activity') instance2_debug = instance2.debug() assert instance2_debug['inbox'] == 4 # An Follow, Accept and Create and Delete activity should be there - instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity # Ensure the post has been delete from instance2's stream inbox_stream = instance2.stream_jsonfeed() @@ -359,3 +392,42 @@ def test_post_content_and_boost_unboost(): note = instance1.outbox_get(f'{create_id}/activity') assert 'shares' in note assert len(note['shares']['items']) == 0 + + +def test_post_content_and_post_reply(): + instance1, instance2 = _instances() + # Instance1 follows instance2 + instance1.follow(instance2) + instance2.follow(instance1) + + inbox_stream = instance2.stream_jsonfeed() + assert len(inbox_stream['items']) == 0 + + instance1_create_id = instance1.new_note('hello') + instance2_debug = instance2.debug() + assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there + assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + + # Ensure the post is visible in instance2's stream + instance2_inbox_stream = instance2.stream_jsonfeed() + assert len(instance2_inbox_stream['items']) == 1 + assert instance2_inbox_stream['items'][0]['id'] == instance1_create_id + + instance2_create_id = instance2.new_note(f'hey @instance1@{instance1.docker_url}', reply=f'{instance1_create_id}/activity') + instance2_debug = instance2.debug() + assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there + assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity + + instance1_debug = instance1.debug() + assert instance1_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there + assert instance1_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity + + instance1_inbox_stream = instance1.stream_jsonfeed() + assert len(instance1_inbox_stream['items']) == 1 + assert instance1_inbox_stream['items'][0]['id'] == instance2_create_id + + # TODO(tsileo): find the activity and check the `replies` collection + + +# TODO(tsileo): +# def test_post_content_and_post_reply_and_delete(): From 559c65f4744cf41c284d4c508769326f3f7c0ca0 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 29 May 2018 18:59:37 +0200 Subject: [PATCH 0058/1425] More tests --- activitypub.py | 150 ++++++++++++++++++++++++++++----------- app.py | 22 ++++-- requirements.txt | 1 + tests/federation_test.py | 54 +++++++++++++- 4 files changed, 179 insertions(+), 48 deletions(-) diff --git a/activitypub.py b/activitypub.py index 7610a69..e4a64f3 100644 --- a/activitypub.py +++ b/activitypub.py @@ -107,7 +107,7 @@ class BaseActivity(object): # Initialize the object self._data: Dict[str, Any] = {'type': self.ACTIVITY_TYPE.value} - logger.debug(f'initializing a {self.ACTIVITY_TYPE.value} activity') + logger.debug(f'initializing a {self.ACTIVITY_TYPE.value} activity: {kwargs}') if 'id' in kwargs: self._data['id'] = kwargs.pop('id') @@ -687,15 +687,22 @@ class Delete(BaseActivity): ACTIVITY_TYPE = ActivityType.DELETE ALLOWED_OBJECT_TYPES = [ActivityType.NOTE, ActivityType.TOMBSTONE] - def _recipients(self) -> List[str]: + def _get_actual_object(self) -> BaseActivity: obj = self.get_object() if obj.type_enum == ActivityType.TOMBSTONE: obj = parse_activity(OBJECT_SERVICE.get(obj.id)) + return obj + + def _recipients(self) -> List[str]: + obj = self._get_actual_object() return obj._recipients() def _process_from_inbox(self) -> None: DB.inbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) - # TODO(tsileo): also delete copies stored in parents' `meta.replies` + obj = self._get_actual_object() + if obj.type_enum == ActivityType.NOTE: + obj._delete_from_threads() + # TODO(tsileo): also purge the cache if it's a reply of a published activity def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: @@ -773,36 +780,60 @@ class Create(BaseActivity): return recipients - def _process_from_inbox(self): + def _update_threads(self) -> None: + logger.debug('_update_threads hook') obj = self.get_object() - tasks.fetch_og.delay('INBOX', self.id) + # TODO(tsileo): re-enable me + # tasks.fetch_og.delay('INBOX', self.id) - in_reply_to = obj.inReplyTo - if in_reply_to: - parent = DB.inbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) - if not parent: - DB.outbox.update_one( - {'activity.object.id': in_reply_to}, - {'$inc': {'meta.count_reply': 1}}, - ) - return + threads = [] + reply = obj.get_local_reply() + logger.debug(f'initial_reply={reply}') + reply_id = None + direct_reply = 1 + while reply is not None: + if not DB.inbox.find_one_and_update({'activity.object.id': reply.id}, { + '$inc': { + 'meta.count_reply': 1, + 'meta.count_direct_reply': direct_reply, + }, + }): + DB.outbox.update_one({'activity.object.id': reply.id}, { + '$inc': { + 'meta.count_reply': 1, + 'meta.count_direct_reply': direct_reply, + }, + }) - # If the note is a "reply of a reply" update the parent message - # TODO(tsileo): review this code - while parent: - DB.inbox.update_one({'_id': parent['_id']}, {'$push': {'meta.replies': self.to_dict()}}) - in_reply_to = parent.get('activity', {}).get('object', {}).get('inReplyTo') - if in_reply_to: - parent = DB.inbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) - if parent is None: - # The reply is a note from the outbox - DB.outbox.update_one( - {'activity.object.id': in_reply_to}, - {'$inc': {'meta.count_reply': 1}}, - ) - else: - parent = None + direct_reply = 0 + reply_id = reply.id + reply = reply.get_local_reply() + logger.debug(f'next_reply={reply}') + if reply: + # Only append to threads if it's not the root + threads.append(reply_id) + + if reply_id: + if not DB.inbox.find_one_and_update({'activity.object.id': obj.id}, { + '$set': { + 'meta.thread_parents': threads, + 'meta.thread_root_parent': reply_id, + }, + }): + DB.outbox.update_one({'activity.object.id': obj.id}, { + '$set': { + 'meta.thread_parents': threads, + 'meta.thread_root_parent': reply_id, + }, + }) + logger.debug('_update_threads done') + + def _process_from_inbox(self) -> None: + self._update_threads() + + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: + self._update_threads() def _should_purge_cache(self) -> bool: # TODO(tsileo): handle reply of a reply... @@ -828,17 +859,9 @@ class Note(BaseActivity): # Remove the `actor` field as `attributedTo` is used for `Note` instead if 'actor' in self._data: del(self._data['actor']) - # FIXME(tsileo): use kwarg - # TODO(tsileo): support mention tag - # TODO(tisleo): implement the tag endpoint if 'sensitive' not in kwargs: self._data['sensitive'] = False - # FIXME(tsileo): add the tag in CC - # for t in kwargs.get('tag', []): - # if t['type'] == 'Mention': - # cc -> c['href'] - def _recipients(self) -> List[str]: # TODO(tsileo): audience support? recipients: List[str] = [] @@ -855,6 +878,51 @@ class Note(BaseActivity): return recipients + def _delete_from_threads(self) -> None: + logger.debug('_delete_from_threads hook') + + reply = self.get_local_reply() + logger.debug(f'initial_reply={reply}') + direct_reply = -1 + while reply is not None: + if not DB.inbox.find_one_and_update({'activity.object.id': reply.id}, { + '$inc': { + 'meta.count_reply': -1, + 'meta.count_direct_reply': direct_reply, + }, + }): + DB.outbox.update_one({'activity.object.id': reply.id}, { + '$inc': { + 'meta.count_reply': 1, + 'meta.count_direct_reply': direct_reply, + }, + }) + + direct_reply = 0 + reply = reply.get_local_reply() + logger.debug(f'next_reply={reply}') + + logger.debug('_delete_from_threads done') + return None + + def get_local_reply(self) -> Optional[BaseActivity]: + "Find the note reply if any.""" + in_reply_to = self.inReplyTo + if not in_reply_to: + # This is the root comment + return None + + inbox_parent = DB.inbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) + if inbox_parent: + return parse_activity(inbox_parent['activity']['object']) + + outbox_parent = DB.outbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) + if outbox_parent: + return parse_activity(outbox_parent['activity']['object']) + + # The parent is no stored on this instance + return None + def build_create(self) -> BaseActivity: """Wraps an activity in a Create activity.""" create_payload = { @@ -872,10 +940,10 @@ class Note(BaseActivity): def build_announce(self) -> BaseActivity: return Announce( - object=self.id, - to=[AS_PUBLIC], - cc=[ID+'/followers', self.attributedTo], - published=datetime.utcnow().replace(microsecond=0).isoformat() + 'Z', + object=self.id, + to=[AS_PUBLIC], + cc=[ID+'/followers', self.attributedTo], + published=datetime.utcnow().replace(microsecond=0).isoformat() + 'Z', ) def build_delete(self) -> BaseActivity: diff --git a/app.py b/app.py index 9d86289..3d15b95 100644 --- a/app.py +++ b/app.py @@ -30,6 +30,7 @@ from passlib.hash import bcrypt from u2flib_server import u2f from urllib.parse import urlparse, urlencode from werkzeug.utils import secure_filename +from flask_wtf.csrf import CSRFProtect import activitypub import config @@ -57,10 +58,17 @@ from utils.key import get_secret_key from utils.webfinger import get_remote_follow_template from utils.webfinger import get_actor_url + + + from typing import Dict, Any app = Flask(__name__) app.secret_key = get_secret_key('flask') +app.config.update( + WTF_CSRF_CHECK_DEFAULT=False, +) +csrf = CSRFProtect(app) logger = logging.getLogger(__name__) @@ -441,15 +449,21 @@ def webfinger(): def add_extra_collection(raw_doc: Dict[str, Any]) -> Dict[str, Any]: + if raw_doc['activity']['type'] != ActivityType.CREATE.value: + return raw_doc + if 'col_likes' in raw_doc.get('meta', {}): col_likes = raw_doc['meta']['col_likes'] - if raw_doc['activity']['type'] == ActivityType.CREATE.value: - raw_doc['activity']['object']['likes'] = embed_collection(col_likes) + raw_doc['activity']['object']['likes'] = embed_collection(col_likes) + if 'col_shares' in raw_doc.get('meta', {}): col_shares = raw_doc['meta']['col_shares'] - if raw_doc['activity']['type'] == ActivityType.CREATE.value: - raw_doc['activity']['object']['shares'] = embed_collection(col_shares) + raw_doc['activity']['object']['shares'] = embed_collection(col_shares) + if 'count_direct_reply' in raw_doc.get('meta', {}): + # FIXME(tsileo): implements the collection handler + raw_doc['activity']['object']['replies'] = {'type': 'Collection', 'totalItems': raw_doc['meta']['count_direct_reply']} + return raw_doc diff --git a/requirements.txt b/requirements.txt index 3e770ad..425405f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ requests markdown python-u2flib-server Flask +Flask-WTF Celery pymongo pyld diff --git a/tests/federation_test.py b/tests/federation_test.py index a5bd1cf..d65c35d 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -297,6 +297,7 @@ def test_post_content_and_like(): assert len(note['likes']['items']) == 1 assert note['likes']['items'][0]['id'] == like_id + def test_post_content_and_like_unlike(): instance1, instance2 = _instances() # Instance1 follows instance2 @@ -332,6 +333,7 @@ def test_post_content_and_like_unlike(): assert 'likes' in note assert len(note['likes']['items']) == 0 + def test_post_content_and_boost(): instance1, instance2 = _instances() # Instance1 follows instance2 @@ -426,8 +428,54 @@ def test_post_content_and_post_reply(): assert len(instance1_inbox_stream['items']) == 1 assert instance1_inbox_stream['items'][0]['id'] == instance2_create_id - # TODO(tsileo): find the activity and check the `replies` collection + instance1_note = instance1.outbox_get(f'{instance1_create_id}/activity') + assert 'replies' in instance1_note + assert instance1_note['replies']['totalItems'] == 1 + # TODO(tsileo): inspect the `replies` collection -# TODO(tsileo): -# def test_post_content_and_post_reply_and_delete(): +def test_post_content_and_post_reply_and_delete(): + instance1, instance2 = _instances() + # Instance1 follows instance2 + instance1.follow(instance2) + instance2.follow(instance1) + + inbox_stream = instance2.stream_jsonfeed() + assert len(inbox_stream['items']) == 0 + + instance1_create_id = instance1.new_note('hello') + instance2_debug = instance2.debug() + assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there + assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + + # Ensure the post is visible in instance2's stream + instance2_inbox_stream = instance2.stream_jsonfeed() + assert len(instance2_inbox_stream['items']) == 1 + assert instance2_inbox_stream['items'][0]['id'] == instance1_create_id + + instance2_create_id = instance2.new_note(f'hey @instance1@{instance1.docker_url}', reply=f'{instance1_create_id}/activity') + instance2_debug = instance2.debug() + assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there + assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity + + instance1_debug = instance1.debug() + assert instance1_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there + assert instance1_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity + + instance1_inbox_stream = instance1.stream_jsonfeed() + assert len(instance1_inbox_stream['items']) == 1 + assert instance1_inbox_stream['items'][0]['id'] == instance2_create_id + + instance1_note = instance1.outbox_get(f'{instance1_create_id}/activity') + assert 'replies' in instance1_note + assert instance1_note['replies']['totalItems'] == 1 + + instance2.delete(f'{instance2_create_id}/activity') + + instance1_debug = instance1.debug() + assert instance1_debug['inbox'] == 4 # An Follow, Accept and Create and Delete activity should be there + assert instance1_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity + + instance1_note = instance1.outbox_get(f'{instance1_create_id}/activity') + assert 'replies' in instance1_note + assert instance1_note['replies']['totalItems'] == 0 From 6aea610fb6cd93e1913366654957a73097529027 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 29 May 2018 21:36:05 +0200 Subject: [PATCH 0059/1425] Lot of cleanup --- README.md | 26 +++++++++++++ activitypub.py | 11 ++++-- app.py | 80 ++++++++++++++++++++++++++++++++-------- tests/federation_test.py | 11 +++++- utils/actor_service.py | 4 ++ utils/errors.py | 21 ++++++++++- utils/key.py | 8 +++- utils/object_service.py | 4 ++ utils/urlutils.py | 3 +- 9 files changed, 143 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 5784e93..21e8eab 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,32 @@ $ docker-compose -f docker-compose-dev.yml up -d $ MICROBLOGPUB_DEBUG=1 FLASK_APP=app.py flask run -p 5005 --with-threads ``` +## User API + +The user API is used by the admin UI (and requires a CSRF token when used with a regular user session), but it can also be accessed with an API key. + +### POST /api/note/delete{?id} + +Delete the given note `id`. + +Answers a **201** (Created) status code. + +You can pass the `id` via JSON, form data or query argument. + +#### Example + +```shell +$ http POST https://microblog.pub/api/note/delete Authorization:'Bearer ' id=http://microblob.pub/outbox//activity +``` + +#### Response + +```json +{ + "activity": "https://microblog.pub/outbox/" +} +``` + ## Contributions PRs are welcome, please open an issue to start a discussion before your start any work. diff --git a/activitypub.py b/activitypub.py index e4a64f3..b63af90 100644 --- a/activitypub.py +++ b/activitypub.py @@ -968,13 +968,18 @@ _ACTIVITY_TYPE_TO_CLS = { } -def parse_activity(payload: ObjectType) -> BaseActivity: +def parse_activity(payload: ObjectType, expected: Optional[ActivityType] = None) -> BaseActivity: t = ActivityType(payload['type']) + + if expected and t != expected: + raise UnexpectedActivityTypeError(f'expected a {expected.name} activity, got a {payload["type"]}') + if t not in _ACTIVITY_TYPE_TO_CLS: - raise ValueError('unsupported activity type') + raise BadActivityTypeError(f'unsupported activity type {payload["type"]}') - return _ACTIVITY_TYPE_TO_CLS[t](**payload) + activity = _ACTIVITY_TYPE_TO_CLS[t](**payload) + return activity def gen_feed(): fg = FeedGenerator() diff --git a/app.py b/app.py index 3d15b95..164e098 100644 --- a/app.py +++ b/app.py @@ -57,8 +57,9 @@ from utils.httpsig import HTTPSigAuth, verify_request from utils.key import get_secret_key from utils.webfinger import get_remote_follow_template from utils.webfinger import get_actor_url - - +from utils.errors import Error +from utils.errors import UnexpectedActivityTypeError +from utils.errors import BadActivityError from typing import Dict, Any @@ -81,8 +82,10 @@ root_logger.setLevel(gunicorn_logger.level) JWT_SECRET = get_secret_key('jwt') JWT = JSONWebSignatureSerializer(JWT_SECRET) -with open('config/jwt_token', 'wb+') as f: - f.write(JWT.dumps({'type': 'admin_token'})) # type: ignore +def _admin_jwt_token() -> str: + return JWT.dumps({'me': 'ADMIN', 'ts': datetime.now().timestamp()}).decode('utf-8') + +ADMIN_API_KEY = get_secret_key('admin_api_key', _admin_jwt_token) SIG_AUTH = HTTPSigAuth(ID+'#main-key', KEY.privkey) @@ -208,15 +211,20 @@ def login_required(f): def _api_required(): if session.get('logged_in'): + #if request.method not in ['GET', 'HEAD']: + # # If a standard API request is made with a "login session", it must havw a CSRF token + # csrf.protect() return # Token verification token = request.headers.get('Authorization', '').replace('Bearer ', '') if not token: + # IndieAuth token token = request.form.get('access_token', '') # Will raise a BadSignature on bad auth payload = JWT.loads(token) + logger.info(f'api call by {payload}') def api_required(f): @@ -249,6 +257,23 @@ def is_api_request(): return True return False + +@app.errorhandler(ValueError) +def handle_value_error(error): + logger.error(f'caught value error: {error!r}') + response = flask_jsonify(message=error.args[0]) + response.status_code = 400 + return response + + +@app.errorhandler(Error) +def handle_activitypub_error(error): + logger.error(f'caught activitypub error {error!r}') + response = flask_jsonify(error.to_dict()) + response.status_code = error.status_code + return response + + # App routes ####### @@ -636,20 +661,43 @@ def notifications(): ) -@app.route('/api/delete') -@api_required -def api_delete(): - # FIXME(tsileo): ensure a Note and not a Create is given - oid = request.args.get('id') - obj = activitypub.parse_activity(OBJECT_SERVICE.get(oid)) - delete = obj.build_delete() - delete.post_to_outbox() +@app.route('/api/key') +@login_required +def api_user_key(): + return flask_jsonify(api_key=ADMIN_API_KEY) + + +def _user_api_get_note(): + if request.is_json(): + oid = request.json.get('id') + else: + oid = request.args.get('id') or request.form.get('id') + + if not oid: + raise ValueError('missing id') + + return activitypub.parse_activity(OBJECT_SERVICE.get(oid), expected=ActivityType.NOTE) + + +def _user_api_response(**kwargs): if request.args.get('redirect'): return redirect(request.args.get('redirect')) - return Response( - status=201, - headers={'Microblogpub-Created-Activity': delete.id}, - ) + + resp = flask_jsonify(**kwargs) + resp.status_code = 201 + return resp + + +@app.route('/api/note/delete', methods=['POST']) +@api_required +def api_delete(): + """API endpoint to delete a Note activity.""" + note = _user_api_get_note() + delete = note.build_delete() + delete.post_to_outbox() + + return _user_api_response(activity=delete.id) + @app.route('/api/boost') @api_required diff --git a/tests/federation_test.py b/tests/federation_test.py index d65c35d..bc6045c 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -19,6 +19,7 @@ class Instance(object): self.docker_url = docker_url or host_url self.session = requests.Session() self._create_delay = 10 + self._auth_headers = {} def _do_req(self, url, headers): url = url.replace(self.docker_url, self.host_url) @@ -51,6 +52,8 @@ class Instance(object): resp = self.session.post(f'{self.host_url}/login', data={'pass': 'hello'}) resp.raise_for_status() assert resp.status_code == 200 + api_key = self.session.get(f'{self.host_url}/api/key').json().get('api_key') + self._auth_headers = {'Authorization': f'Bearer {api_key}'} def block(self, actor_url) -> None: # Instance1 follows instance2 @@ -95,11 +98,15 @@ class Instance(object): return resp.headers.get('microblogpub-created-activity') def delete(self, oid: str) -> None: - resp = self.session.get(f'{self.host_url}/api/delete', params={'id': oid}) + resp = requests.post( + f'{self.host_url}/api/note/delete', + json={'id': oid}, + headers=self._auth_headers, + ) assert resp.status_code == 201 time.sleep(self._create_delay) - return resp.headers.get('microblogpub-created-activity') + return resp.json().get('activity') def undo(self, oid: str) -> None: resp = self.session.get(f'{self.host_url}/api/undo', params={'id': oid}) diff --git a/utils/actor_service.py b/utils/actor_service.py index 6c1f35a..9982235 100644 --- a/utils/actor_service.py +++ b/utils/actor_service.py @@ -5,6 +5,7 @@ from urllib.parse import urlparse from Crypto.PublicKey import RSA from .urlutils import check_url +from .errors import ActivityNotFoundError logger = logging.getLogger(__name__) @@ -32,6 +33,9 @@ class ActorService(object): 'Accept': 'application/activity+json', 'User-Agent': self._user_agent, }) + if resp.status_code == 404: + raise ActivityNotFoundError(f'{actor_url} cannot be fetched, 404 not found error') + resp.raise_for_status() return resp.json() diff --git a/utils/errors.py b/utils/errors.py index 31e678e..5356267 100644 --- a/utils/errors.py +++ b/utils/errors.py @@ -1,6 +1,25 @@ class Error(Exception): - pass + status_code = 400 + + def __init__(self, message, status_code=None, payload=None): + Exception.__init__(self) + self.message = message + if status_code is not None: + self.status_code = status_code + self.payload = payload + + def to_dict(self): + rv = dict(self.payload or ()) + rv['message'] = self.message + return rv + + def __repr__(self): + return f'{self.__class__.__qualname__}({self.message!r}, payload={self.payload!r}, status_code={self.status_code})' + + +class ActivityNotFoundError(Error): + status_code = 404 class BadActivityError(Error): diff --git a/utils/key.py b/utils/key.py index 526b3be..e101af7 100644 --- a/utils/key.py +++ b/utils/key.py @@ -2,14 +2,18 @@ import os import binascii from Crypto.PublicKey import RSA +from typing import Callable KEY_DIR = 'config/' -def get_secret_key(name:str) -> str: +def _new_key() -> str: + return binascii.hexlify(os.urandom(32)).decode('utf-8') + +def get_secret_key(name: str, new_key: Callable[[], str] = _new_key) -> str: key_path = f'{KEY_DIR}{name}.key' if not os.path.exists(key_path): - k = binascii.hexlify(os.urandom(32)).decode('utf-8') + k = new_key() with open(key_path, 'w+') as f: f.write(k) return k diff --git a/utils/object_service.py b/utils/object_service.py index 9445550..1ebc0ce 100644 --- a/utils/object_service.py +++ b/utils/object_service.py @@ -2,6 +2,7 @@ import requests from urllib.parse import urlparse from .urlutils import check_url +from .errors import ActivityNotFoundError class ObjectService(object): @@ -20,6 +21,9 @@ class ObjectService(object): 'Accept': 'application/activity+json', 'User-Agent': self._user_agent, }) + if resp.status_code == 404: + raise ActivityNotFoundError(f'{object_id} cannot be fetched, 404 error not found') + resp.raise_for_status() return resp.json() diff --git a/utils/urlutils.py b/utils/urlutils.py index be37c99..99f900d 100644 --- a/utils/urlutils.py +++ b/utils/urlutils.py @@ -5,11 +5,12 @@ import ipaddress from urllib.parse import urlparse from . import strtobool +from .errors import Error logger = logging.getLogger(__name__) -class InvalidURLError(Exception): +class InvalidURLError(Error): pass From 48ab7137abe34f4ea6ad818da0f3eb4c954087b2 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 29 May 2018 21:47:28 +0200 Subject: [PATCH 0060/1425] Fix type annotation and README --- README.md | 4 +++- activitypub.py | 3 ++- app.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 21e8eab..7aa1bea 100644 --- a/README.md +++ b/README.md @@ -91,9 +91,11 @@ $ MICROBLOGPUB_DEBUG=1 FLASK_APP=app.py flask run -p 5005 --with-threads The user API is used by the admin UI (and requires a CSRF token when used with a regular user session), but it can also be accessed with an API key. +All the examples are using [HTTPie](https://httpie.org/). + ### POST /api/note/delete{?id} -Delete the given note `id`. +Deletes the given note `id`. Answers a **201** (Created) status code. diff --git a/activitypub.py b/activitypub.py index b63af90..1534a78 100644 --- a/activitypub.py +++ b/activitypub.py @@ -975,12 +975,13 @@ def parse_activity(payload: ObjectType, expected: Optional[ActivityType] = None) raise UnexpectedActivityTypeError(f'expected a {expected.name} activity, got a {payload["type"]}') if t not in _ACTIVITY_TYPE_TO_CLS: - raise BadActivityTypeError(f'unsupported activity type {payload["type"]}') + raise BadActivityError(f'unsupported activity type {payload["type"]}') activity = _ACTIVITY_TYPE_TO_CLS[t](**payload) return activity + def gen_feed(): fg = FeedGenerator() fg.id(f'{ID}') diff --git a/app.py b/app.py index 164e098..f305fa2 100644 --- a/app.py +++ b/app.py @@ -83,7 +83,7 @@ JWT_SECRET = get_secret_key('jwt') JWT = JSONWebSignatureSerializer(JWT_SECRET) def _admin_jwt_token() -> str: - return JWT.dumps({'me': 'ADMIN', 'ts': datetime.now().timestamp()}).decode('utf-8') + return JWT.dumps({'me': 'ADMIN', 'ts': datetime.now().timestamp()}).decode('utf-8') # type: ignore ADMIN_API_KEY = get_secret_key('admin_api_key', _admin_jwt_token) From a148ec6068757d5a59ebdc85ca1a24adf5211453 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 29 May 2018 22:16:09 +0200 Subject: [PATCH 0061/1425] Bugfix --- .travis.yml | 1 + app.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8477299..117f6f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,3 +25,4 @@ script: - python -m pytest -v --ignore data -k integration # Federation tests (with two local instances) - python -m pytest -v -s --ignore data -k federation + - docker logs instance1_web_1 diff --git a/app.py b/app.py index f305fa2..559ded3 100644 --- a/app.py +++ b/app.py @@ -69,7 +69,7 @@ app.secret_key = get_secret_key('flask') app.config.update( WTF_CSRF_CHECK_DEFAULT=False, ) -csrf = CSRFProtect(app) +# csrf = CSRFProtect(app) logger = logging.getLogger(__name__) @@ -668,7 +668,7 @@ def api_user_key(): def _user_api_get_note(): - if request.is_json(): + if request.is_json: oid = request.json.get('id') else: oid = request.args.get('id') or request.form.get('id') From b7512dca08203cd18818e162a2fb37d42e304a0e Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 30 May 2018 21:55:57 +0200 Subject: [PATCH 0062/1425] Read the API key from config in the federation tests --- tests/federation_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/federation_test.py b/tests/federation_test.py index bc6045c..12ead00 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -14,12 +14,14 @@ def resp2plaintext(resp): class Instance(object): """Test instance wrapper.""" - def __init__(self, host_url, docker_url=None): + def __init__(self, name, host_url, docker_url=None): self.host_url = host_url self.docker_url = docker_url or host_url self.session = requests.Session() self._create_delay = 10 - self._auth_headers = {} + with open(f'tests/fixtures/{name}/config/admin_api_key.key') as f: + api_key = f.read() + self._auth_headers = {'Authorization': f'Bearer {api_key}'} def _do_req(self, url, headers): url = url.replace(self.docker_url, self.host_url) @@ -52,8 +54,6 @@ class Instance(object): resp = self.session.post(f'{self.host_url}/login', data={'pass': 'hello'}) resp.raise_for_status() assert resp.status_code == 200 - api_key = self.session.get(f'{self.host_url}/api/key').json().get('api_key') - self._auth_headers = {'Authorization': f'Bearer {api_key}'} def block(self, actor_url) -> None: # Instance1 follows instance2 @@ -149,10 +149,10 @@ class Instance(object): def _instances(): - instance1 = Instance('http://localhost:5006', 'http://instance1_web_1:5005') + instance1 = Instance('instance1', 'http://localhost:5006', 'http://instance1_web_1:5005') instance1.ping() - instance2 = Instance('http://localhost:5007', 'http://instance2_web_1:5005') + instance2 = Instance('instance2', 'http://localhost:5007', 'http://instance2_web_1:5005') instance2.ping() # Login From 69a5ceb455f34b9af4f9fb2a2b88a5bf0a489dbf Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 30 May 2018 22:27:07 +0200 Subject: [PATCH 0063/1425] Cache the JSON-LD schemas --- utils/linked_data_sig.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/utils/linked_data_sig.py b/utils/linked_data_sig.py index 9523ed4..834c9bd 100644 --- a/utils/linked_data_sig.py +++ b/utils/linked_data_sig.py @@ -6,6 +6,23 @@ from Crypto.Signature import PKCS1_v1_5 from Crypto.Hash import SHA256 import base64 +from typing import Any, Dict + + +# cache the downloaded "schemas", otherwise the library is super slow +# (https://github.com/digitalbazaar/pyld/issues/70) +_CACHE: Dict[str, Any] = {} +LOADER = jsonld.requests_document_loader() + +def _caching_document_loader(url: str) -> Any: + if url in _CACHE: + return _CACHE[url] + resp = LOADER(url) + _CACHE[url] = resp + return resp + +jsonld.set_document_loader(_caching_document_loader) + def options_hash(doc): doc = dict(doc['signature']) From d2be270ccbf7bc1f16d6ab13ebaa70a489f26a4c Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 30 May 2018 22:36:04 +0200 Subject: [PATCH 0064/1425] Tweak the tests --- .travis.yml | 1 + tests/federation_test.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 117f6f2..60dd487 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,5 +24,6 @@ script: # Integration tests first - python -m pytest -v --ignore data -k integration # Federation tests (with two local instances) + - ls -lR tests - python -m pytest -v -s --ignore data -k federation - docker logs instance1_web_1 diff --git a/tests/federation_test.py b/tests/federation_test.py index 12ead00..053b6ec 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -19,7 +19,7 @@ class Instance(object): self.docker_url = docker_url or host_url self.session = requests.Session() self._create_delay = 10 - with open(f'tests/fixtures/{name}/config/admin_api_key.key') as f: + with open(os.path.join(os.path.abspath(__file__), f'fixtures/{name}/config/admin_api_key.key')) as f: api_key = f.read() self._auth_headers = {'Authorization': f'Bearer {api_key}'} From 3dbe4ba4e9d089ae91327359978db8d7a84448f6 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 30 May 2018 22:50:45 +0200 Subject: [PATCH 0065/1425] Debug the CI --- .travis.yml | 2 +- config.py | 4 ++-- tests/federation_test.py | 2 +- utils/key.py | 4 +++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 60dd487..c3f7aa0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,6 @@ script: # Integration tests first - python -m pytest -v --ignore data -k integration # Federation tests (with two local instances) - - ls -lR tests + - ls -lR /app - python -m pytest -v -s --ignore data -k federation - docker logs instance1_web_1 diff --git a/config.py b/config.py index e4c3912..b1645df 100644 --- a/config.py +++ b/config.py @@ -5,7 +5,7 @@ from pymongo import MongoClient import requests from utils import strtobool -from utils.key import Key +from utils.key import Key, KEY_DIR from utils.actor_service import ActorService from utils.object_service import ObjectService @@ -35,7 +35,7 @@ HEADERS = [ ] -with open('config/me.yml') as f: +with open(os.path.join(KEY_DIR, 'me.yml')) as f: conf = yaml.load(f) USERNAME = conf['username'] diff --git a/tests/federation_test.py b/tests/federation_test.py index 053b6ec..c681dc1 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -19,7 +19,7 @@ class Instance(object): self.docker_url = docker_url or host_url self.session = requests.Session() self._create_delay = 10 - with open(os.path.join(os.path.abspath(__file__), f'fixtures/{name}/config/admin_api_key.key')) as f: + with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), f'fixtures/{name}/config/admin_api_key.key')) as f: api_key = f.read() self._auth_headers = {'Authorization': f'Bearer {api_key}'} diff --git a/utils/key.py b/utils/key.py index e101af7..f974aac 100644 --- a/utils/key.py +++ b/utils/key.py @@ -4,7 +4,9 @@ import binascii from Crypto.PublicKey import RSA from typing import Callable -KEY_DIR = 'config/' +KEY_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), '..', 'config' +) def _new_key() -> str: From c53913aa121958296324279a43fef59848c2c087 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 30 May 2018 23:09:45 +0200 Subject: [PATCH 0066/1425] Debug CI --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c3f7aa0..bdaa9e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,7 @@ script: # Integration tests first - python -m pytest -v --ignore data -k integration # Federation tests (with two local instances) - - ls -lR /app + - pwd + - ls -lR . - python -m pytest -v -s --ignore data -k federation - docker logs instance1_web_1 From dc1260f56435fb641786fbc443ac71d725a928db Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 30 May 2018 23:13:10 +0200 Subject: [PATCH 0067/1425] Debug CI --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index bdaa9e1..0e06e39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,7 @@ script: - docker-compose ps - WEB_PORT=5006 CONFIG_DIR=./tests/fixtures/instance1/config docker-compose -p instance1 -f docker-compose-tests.yml up -d - docker-compose -p instance1 -f docker-compose-tests.yml ps + - docker inspect instance1_web_1 - WEB_PORT=5007 CONFIG_DIR=./tests/fixtures/instance2/config docker-compose -p instance2 -f docker-compose-tests.yml up -d - docker-compose -p instance2 -f docker-compose-tests.yml ps # Integration tests first From 0932b98a442fc56f4e83828f1db0e1996e33c0fa Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 30 May 2018 23:35:02 +0200 Subject: [PATCH 0068/1425] Debug CI --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0e06e39..e54d23b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,8 @@ script: # Integration tests first - python -m pytest -v --ignore data -k integration # Federation tests (with two local instances) + - curl http://localhost:5006 + - curl http://localhost:5007 - pwd - ls -lR . - python -m pytest -v -s --ignore data -k federation From fc1bd190ef73010161cdf07523769b9ecc5162f5 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 30 May 2018 23:47:01 +0200 Subject: [PATCH 0069/1425] Tweak the config --- app.py | 10 ++-------- config.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/app.py b/app.py index 559ded3..1b10749 100644 --- a/app.py +++ b/app.py @@ -51,6 +51,8 @@ from config import PASS from config import HEADERS from config import VERSION from config import DEBUG_MODE +from config import JWT +from config import ADMIN_API_KEY from config import _drop_db from config import custom_cache_purge_hook from utils.httpsig import HTTPSigAuth, verify_request @@ -79,14 +81,6 @@ root_logger = logging.getLogger() root_logger.handlers = gunicorn_logger.handlers root_logger.setLevel(gunicorn_logger.level) -JWT_SECRET = get_secret_key('jwt') -JWT = JSONWebSignatureSerializer(JWT_SECRET) - -def _admin_jwt_token() -> str: - return JWT.dumps({'me': 'ADMIN', 'ts': datetime.now().timestamp()}).decode('utf-8') # type: ignore - -ADMIN_API_KEY = get_secret_key('admin_api_key', _admin_jwt_token) - SIG_AUTH = HTTPSigAuth(ID+'#main-key', KEY.privkey) diff --git a/config.py b/config.py index b1645df..1be18f7 100644 --- a/config.py +++ b/config.py @@ -3,9 +3,10 @@ import os import yaml from pymongo import MongoClient import requests +from itsdangerous import JSONWebSignatureSerializer from utils import strtobool -from utils.key import Key, KEY_DIR +from utils.key import Key, KEY_DIR, get_secret_key from utils.actor_service import ActorService from utils.object_service import ObjectService @@ -73,6 +74,16 @@ def _drop_db(): KEY = Key(USERNAME, DOMAIN, create=True) + +JWT_SECRET = get_secret_key('jwt') +JWT = JSONWebSignatureSerializer(JWT_SECRET) + +def _admin_jwt_token() -> str: + return JWT.dumps({'me': 'ADMIN', 'ts': datetime.now().timestamp()}).decode('utf-8') # type: ignore + +ADMIN_API_KEY = get_secret_key('admin_api_key', _admin_jwt_token) + + ME = { "@context": [ CTX_AS, From 2938191b3a804b1b4b1c5b136636c63d5b9c2c60 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 30 May 2018 23:53:14 +0200 Subject: [PATCH 0070/1425] Fix config --- config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/config.py b/config.py index 1be18f7..506a4d3 100644 --- a/config.py +++ b/config.py @@ -4,6 +4,7 @@ import yaml from pymongo import MongoClient import requests from itsdangerous import JSONWebSignatureSerializer +from datetime import datetime from utils import strtobool from utils.key import Key, KEY_DIR, get_secret_key From 905070b8ea08ca9849872cb68a3a5931a5d17614 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 30 May 2018 23:57:14 +0200 Subject: [PATCH 0071/1425] Fix config --- utils/key.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/key.py b/utils/key.py index f974aac..056ee36 100644 --- a/utils/key.py +++ b/utils/key.py @@ -13,7 +13,7 @@ def _new_key() -> str: return binascii.hexlify(os.urandom(32)).decode('utf-8') def get_secret_key(name: str, new_key: Callable[[], str] = _new_key) -> str: - key_path = f'{KEY_DIR}{name}.key' + key_path = os.path.join(KEY_DIR, f'{name}.key') if not os.path.exists(key_path): k = new_key() with open(key_path, 'w+') as f: @@ -28,7 +28,7 @@ class Key(object): def __init__(self, user: str, domain: str, create: bool = True) -> None: user = user.replace('.', '_') domain = domain.replace('.', '_') - key_path = f'{KEY_DIR}/key_{user}_{domain}.pem' + key_path = os.path.join(KEY_DIR, f'key_{user}_{domain}.pem') if os.path.isfile(key_path): with open(key_path) as f: self.privkey_pem = f.read() From f92954072902ab4d67c66574b94193a9dce5d633 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 31 May 2018 00:02:39 +0200 Subject: [PATCH 0072/1425] Clean the CI script --- .travis.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index e54d23b..8477299 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,15 +19,9 @@ script: - docker-compose ps - WEB_PORT=5006 CONFIG_DIR=./tests/fixtures/instance1/config docker-compose -p instance1 -f docker-compose-tests.yml up -d - docker-compose -p instance1 -f docker-compose-tests.yml ps - - docker inspect instance1_web_1 - WEB_PORT=5007 CONFIG_DIR=./tests/fixtures/instance2/config docker-compose -p instance2 -f docker-compose-tests.yml up -d - docker-compose -p instance2 -f docker-compose-tests.yml ps # Integration tests first - python -m pytest -v --ignore data -k integration # Federation tests (with two local instances) - - curl http://localhost:5006 - - curl http://localhost:5007 - - pwd - - ls -lR . - python -m pytest -v -s --ignore data -k federation - - docker logs instance1_web_1 From 848b8b23a8765d86bab2830c8d65d6b37337d03b Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 31 May 2018 00:08:00 +0200 Subject: [PATCH 0073/1425] Fix build --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 8477299..f613e2d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,8 @@ script: - docker-compose -p instance1 -f docker-compose-tests.yml ps - WEB_PORT=5007 CONFIG_DIR=./tests/fixtures/instance2/config docker-compose -p instance2 -f docker-compose-tests.yml up -d - docker-compose -p instance2 -f docker-compose-tests.yml ps + - curl http://localhost:5006 + - curl http://localhost:5007 # Integration tests first - python -m pytest -v --ignore data -k integration # Federation tests (with two local instances) From 6711722bd0a2729fad548c6d7ed42ad9f0889c6f Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 1 Jun 2018 01:26:23 +0200 Subject: [PATCH 0074/1425] Add new shares/likes/replies collection for notes --- activitypub.py | 40 +++++++-------- app.py | 103 +++++++++++++++++++++++++++++++++++---- tests/federation_test.py | 23 +++++---- 3 files changed, 127 insertions(+), 39 deletions(-) diff --git a/activitypub.py b/activitypub.py index 1534a78..ec14cc3 100644 --- a/activitypub.py +++ b/activitypub.py @@ -380,6 +380,16 @@ class BaseActivity(object): else: try: actor = Person(**ACTOR_SERVICE.get(recipient)) + + if actor.endpoints: + shared_inbox = actor.endpoints.get('sharedInbox') + if shared_inbox not in out: + out.append(shared_inbox) + continue + + if actor.inbox and actor.inbox not in out: + out.append(actor.inbox) + except NotAnActorError as error: # Is the activity a `Collection`/`OrderedCollection`? if error.activity and error.activity['type'] in [ActivityType.COLLECTION.value, @@ -400,17 +410,6 @@ class BaseActivity(object): if col_actor.inbox and col_actor.inbox not in out: out.append(col_actor.inbox) - continue - - if actor.endpoints: - shared_inbox = actor.endpoints.get('sharedInbox') - if shared_inbox not in out: - out.append(shared_inbox) - continue - - if actor.inbox and actor.inbox not in out: - out.append(actor.inbox) - return out def build_undo(self) -> 'BaseActivity': @@ -1076,11 +1075,11 @@ def parse_collection(payload: Optional[Dict[str, Any]] = None, url: Optional[str return activitypub_utils.parse_collection(payload, url) -def embed_collection(data): +def embed_collection(total_items, first_page_id): return { - "type": "Collection", - "totalItems": len(data), - "items": data, + "type": ActivityType.ORDERED_COLLECTION.value, + "totalItems": total_items, + "first": first_page_id, } @@ -1097,7 +1096,7 @@ def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, return { 'id': BASE_URL + '/' + col_name, 'totalItems': 0, - 'type': 'OrderedCollection', + 'type': ActivityType.ORDERED_COLLECTION.value, 'orederedItems': [], } @@ -1115,13 +1114,13 @@ def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, '@context': CTX_AS, 'id': f'{BASE_URL}/{col_name}', 'totalItems': total_items, - 'type': 'OrderedCollection', + 'type': ActivityType.ORDERED_COLLECTION.value, 'first': { 'id': f'{BASE_URL}/{col_name}?cursor={start_cursor}', 'orderedItems': data, 'partOf': f'{BASE_URL}/{col_name}', 'totalItems': total_items, - 'type': 'OrderedCollectionPage' + 'type': ActivityType.ORDERED_COLLECTION_PAGE.value, }, } @@ -1130,10 +1129,11 @@ def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, return resp + # If there's a cursor, then we return an OrderedCollectionPage resp = { '@context': CTX_AS, - 'type': 'OrderedCollectionPage', + 'type': ActivityType.ORDERED_COLLECTION_PAGE.value, 'id': BASE_URL + '/' + col_name + '?cursor=' + start_cursor, 'totalItems': total_items, 'partOf': BASE_URL + '/' + col_name, @@ -1142,4 +1142,6 @@ def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, if len(data) == limit: resp['next'] = BASE_URL + '/' + col_name + '?cursor=' + next_page_cursor + # TODO(tsileo): implements prev with prev= + return resp diff --git a/app.py b/app.py index 1b10749..127c0f5 100644 --- a/app.py +++ b/app.py @@ -471,18 +471,21 @@ def add_extra_collection(raw_doc: Dict[str, Any]) -> Dict[str, Any]: if raw_doc['activity']['type'] != ActivityType.CREATE.value: return raw_doc - if 'col_likes' in raw_doc.get('meta', {}): - col_likes = raw_doc['meta']['col_likes'] - raw_doc['activity']['object']['likes'] = embed_collection(col_likes) + raw_doc['activity']['object']['replies'] = embed_collection( + raw_doc.get('meta', {}).get('count_direct_reply', 0), + f'{ID}/outbox/{raw_doc["id"]}/replies', + ) - if 'col_shares' in raw_doc.get('meta', {}): - col_shares = raw_doc['meta']['col_shares'] - raw_doc['activity']['object']['shares'] = embed_collection(col_shares) + raw_doc['activity']['object']['likes'] = embed_collection( + raw_doc.get('meta', {}).get('count_like', 0), + f'{ID}/outbox/{raw_doc["id"]}/likes', + ) + + raw_doc['activity']['object']['shares'] = embed_collection( + raw_doc.get('meta', {}).get('count_boost', 0), + f'{ID}/outbox/{raw_doc["id"]}/shares', + ) - if 'count_direct_reply' in raw_doc.get('meta', {}): - # FIXME(tsileo): implements the collection handler - raw_doc['activity']['object']['replies'] = {'type': 'Collection', 'totalItems': raw_doc['meta']['count_direct_reply']} - return raw_doc @@ -547,6 +550,86 @@ def outbox_activity(item_id): return jsonify(**obj['object']) +@app.route('/outbox//replies') +def outbox_activity_replies(item_id): + if not is_api_request(): + abort(404) + data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + if not data: + abort(404) + obj = activitypub.parse_activity(data) + if obj.type_enum != ActivityType.CREATE: + abort(404) + + q = { + 'meta.deleted': False, + 'type': ActivityType.CREATE.value, + 'activity.object.inReplyTo': obj.get_object().id, + } + + return jsonify(**activitypub.build_ordered_collection( + DB.inbox, + q=q, + cursor=request.args.get('cursor'), + map_func=lambda doc: doc['activity'], + col_name=f'outbox/{item_id}/replies', + )) + + +@app.route('/outbox//likes') +def outbox_activity_likes(item_id): + if not is_api_request(): + abort(404) + data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + if not data: + abort(404) + obj = activitypub.parse_activity(data) + if obj.type_enum != ActivityType.CREATE: + abort(404) + + q = { + 'meta.undo': False, + 'type': ActivityType.LIKE.value, + '$or': [{'activity.object.id': obj.get_object().id}, + {'activity.object': obj.get_object().id}], + } + + return jsonify(**activitypub.build_ordered_collection( + DB.inbox, + q=q, + cursor=request.args.get('cursor'), + map_func=lambda doc: doc['activity'], + col_name=f'outbox/{item_id}/likes', + )) + + +@app.route('/outbox//shares') +def outbox_activity_shares(item_id): + if not is_api_request(): + abort(404) + data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + if not data: + abort(404) + obj = activitypub.parse_activity(data) + if obj.type_enum != ActivityType.CREATE: + abort(404) + + q = { + 'meta.undo': False, + 'type': ActivityType.ANNOUNCE.value, + '$or': [{'activity.object.id': obj.get_object().id}, + {'activity.object': obj.get_object().id}], + } + + return jsonify(**activitypub.build_ordered_collection( + DB.inbox, + q=q, + cursor=request.args.get('cursor'), + map_func=lambda doc: doc['activity'], + col_name=f'outbox/{item_id}/shares', + )) + + @app.route('/admin', methods=['GET']) @login_required def admin(): diff --git a/tests/federation_test.py b/tests/federation_test.py index c681dc1..9a6c5a1 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -301,8 +301,8 @@ def test_post_content_and_like(): note = instance1.outbox_get(f'{create_id}/activity') assert 'likes' in note - assert len(note['likes']['items']) == 1 - assert note['likes']['items'][0]['id'] == like_id + assert note['likes']['totalItems'] == 1 + # assert note['likes']['items'][0]['id'] == like_id def test_post_content_and_like_unlike(): @@ -327,8 +327,9 @@ def test_post_content_and_like_unlike(): note = instance1.outbox_get(f'{create_id}/activity') assert 'likes' in note - assert len(note['likes']['items']) == 1 - assert note['likes']['items'][0]['id'] == like_id + assert note['likes']['totalItems'] == 1 + # FIXME(tsileo): parse the collection + # assert note['likes']['items'][0]['id'] == like_id instance2.undo(like_id) @@ -338,7 +339,7 @@ def test_post_content_and_like_unlike(): note = instance1.outbox_get(f'{create_id}/activity') assert 'likes' in note - assert len(note['likes']['items']) == 0 + assert note['likes']['totalItems'] == 0 def test_post_content_and_boost(): @@ -363,8 +364,9 @@ def test_post_content_and_boost(): note = instance1.outbox_get(f'{create_id}/activity') assert 'shares' in note - assert len(note['shares']['items']) == 1 - assert note['shares']['items'][0]['id'] == boost_id + assert note['shares']['totalItems'] == 1 + # FIXME(tsileo): parse the collection + # assert note['shares']['items'][0]['id'] == boost_id def test_post_content_and_boost_unboost(): @@ -389,8 +391,9 @@ def test_post_content_and_boost_unboost(): note = instance1.outbox_get(f'{create_id}/activity') assert 'shares' in note - assert len(note['shares']['items']) == 1 - assert note['shares']['items'][0]['id'] == boost_id + assert note['shares']['totalItems'] == 1 + # FIXME(tsileo): parse the collection + # assert note['shares']['items'][0]['id'] == boost_id instance2.undo(boost_id) @@ -400,7 +403,7 @@ def test_post_content_and_boost_unboost(): note = instance1.outbox_get(f'{create_id}/activity') assert 'shares' in note - assert len(note['shares']['items']) == 0 + assert note['shares']['totalItems'] == 0 def test_post_content_and_post_reply(): From 45afd99098a6b2507e5e60db9bfca505c88eb5bc Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 1 Jun 2018 01:28:32 +0200 Subject: [PATCH 0075/1425] Fix flake8 warning --- activitypub.py | 1 - 1 file changed, 1 deletion(-) diff --git a/activitypub.py b/activitypub.py index ec14cc3..1ec068d 100644 --- a/activitypub.py +++ b/activitypub.py @@ -1129,7 +1129,6 @@ def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, return resp - # If there's a cursor, then we return an OrderedCollectionPage resp = { '@context': CTX_AS, From f8ee19b4d16ad8ecec75abe10203da65755e1312 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 1 Jun 2018 20:29:44 +0200 Subject: [PATCH 0076/1425] User API cleanup --- .travis.yml | 1 + Makefile | 11 ++ README.md | 144 ++++++++++++++++++++++++- activitypub.py | 23 ++-- app.py | 174 +++++++++++++++--------------- docker-compose-tests.yml | 4 +- docker-compose.yml | 4 +- tasks.py | 9 +- tests/federation_test.py | 224 +++++++++++++++++++++++++++------------ utils/errors.py | 3 + utils/key.py | 3 +- 11 files changed, 422 insertions(+), 178 deletions(-) diff --git a/.travis.yml b/.travis.yml index f613e2d..a523ee1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ script: - mypy --ignore-missing-imports . - flake8 activitypub.py - cp -r tests/fixtures/me.yml config/me.yml + - docker build . -t microblogpub:latest - docker-compose up -d - docker-compose ps - WEB_PORT=5006 CONFIG_DIR=./tests/fixtures/instance1/config docker-compose -p instance1 -f docker-compose-tests.yml up -d diff --git a/Makefile b/Makefile index 333f991..6221cdd 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,18 @@ css: password: python -c "import bcrypt; from getpass import getpass; print(bcrypt.hashpw(getpass().encode('utf-8'), bcrypt.gensalt()).decode('utf-8'))" +docker: + mypy . --ignore-missing-imports + docker build . -t microblogpub:latest + +reload-fed: + docker-compose -p instance2 -f docker-compose-tests.yml stop + docker-compose -p instance1 -f docker-compose-tests.yml stop + WEB_PORT=5006 CONFIG_DIR=./tests/fixtures/instance1/config docker-compose -p instance1 -f docker-compose-tests.yml up -d --force-recreate --build + WEB_PORT=5007 CONFIG_DIR=./tests/fixtures/instance2/config docker-compose -p instance2 -f docker-compose-tests.yml up -d --force-recreate --build + update: docker-compose stop git pull + docker build . -t microblogpub:latest docker-compose up -d --force-recreate --build diff --git a/README.md b/README.md index 7aa1bea..b8fe115 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,20 @@ $ docker-compose -f docker-compose-dev.yml up -d $ MICROBLOGPUB_DEBUG=1 FLASK_APP=app.py flask run -p 5005 --with-threads ``` +## ActivityPub API + +### GET / + +Returns the actor profile, with links to all the "standard" collections. + +### GET /tags/:tag + +Special collection that reference notes with the given tag. + +### GET /stream + +Special collection that returns the stream/inbox as displayed in the UI. + ## User API The user API is used by the admin UI (and requires a CSRF token when used with a regular user session), but it can also be accessed with an API key. @@ -95,7 +109,7 @@ All the examples are using [HTTPie](https://httpie.org/). ### POST /api/note/delete{?id} -Deletes the given note `id`. +Deletes the given note `id` (the note must from the instance outbox). Answers a **201** (Created) status code. @@ -104,7 +118,7 @@ You can pass the `id` via JSON, form data or query argument. #### Example ```shell -$ http POST https://microblog.pub/api/note/delete Authorization:'Bearer ' id=http://microblob.pub/outbox//activity +$ http POST https://microblog.pub/api/note/delete Authorization:'Bearer ' id=http://microblob.pub/outbox//activity ``` #### Response @@ -115,6 +129,132 @@ $ http POST https://microblog.pub/api/note/delete Authorization:'Bearer ' } ``` +### POST /api/like{?id} + +Likes the given activity. + +Answers a **201** (Created) status code. + +You can pass the `id` via JSON, form data or query argument. + +#### Example + +```shell +$ http POST https://microblog.pub/api/like Authorization:'Bearer ' id=http://activity-iri.tld +``` + +#### Response + +```json +{ + "activity": "https://microblog.pub/outbox/" +} +``` + +### POST /api/boost{?id} + +Boosts/Announces the given activity. + +Answers a **201** (Created) status code. + +You can pass the `id` via JSON, form data or query argument. + +#### Example + +```shell +$ http POST https://microblog.pub/api/boost Authorization:'Bearer ' id=http://activity-iri.tld +``` + +#### Response + +```json +{ + "activity": "https://microblog.pub/outbox/" +} +``` + +### POST /api/block{?actor} + +Blocks the given actor, all activities from this actor will be dropped after that. + +Answers a **201** (Created) status code. + +You can pass the `id` via JSON, form data or query argument. + +#### Example + +```shell +$ http POST https://microblog.pub/api/block Authorization:'Bearer ' actor=http://actor-iri.tld/ +``` + +#### Response + +```json +{ + "activity": "https://microblog.pub/outbox/" +} +``` + +### POST /api/follow{?actor} + +Follows the given actor. + +Answers a **201** (Created) status code. + +You can pass the `id` via JSON, form data or query argument. + +#### Example + +```shell +$ http POST https://microblog.pub/api/follow Authorization:'Bearer ' actor=http://actor-iri.tld/ +``` + +#### Response + +```json +{ + "activity": "https://microblog.pub/outbox/" +} +``` + +### POST /api/new_note{?content,reply} + +Creates a new note. `reply` is the IRI of the "replied" note if any. + +Answers a **201** (Created) status code. + +You can pass the `content` and `reply` via JSON, form data or query argument. + +#### Example + +```shell +$ http POST https://microblog.pub/api/new_note Authorization:'Bearer ' content=hello +``` + +#### Response + +```json +{ + "activity": "https://microblog.pub/outbox/" +} +``` + + +### GET /api/stream + + +#### Example + +```shell +$ http GET https://microblog.pub/api/stream Authorization:'Bearer ' +``` + +#### Response + +```json +``` + + ## Contributions PRs are welcome, please open an issue to start a discussion before your start any work. diff --git a/activitypub.py b/activitypub.py index 1ec068d..1cd048c 100644 --- a/activitypub.py +++ b/activitypub.py @@ -9,13 +9,12 @@ from bson.objectid import ObjectId from html2text import html2text from feedgen.feed import FeedGenerator -from utils.linked_data_sig import generate_signature from utils.actor_service import NotAnActorError from utils.errors import BadActivityError, UnexpectedActivityTypeError from utils import activitypub_utils from config import USERNAME, BASE_URL, ID from config import CTX_AS, CTX_SECURITY, AS_PUBLIC -from config import KEY, DB, ME, ACTOR_SERVICE +from config import DB, ME, ACTOR_SERVICE from config import OBJECT_SERVICE from config import PUBLIC_INSTANCES import tasks @@ -350,7 +349,6 @@ class BaseActivity(object): except NotImplementedError: logger.debug('post to outbox hook not implemented') - generate_signature(activity, KEY.privkey) payload = json.dumps(activity) for recp in recipients: logger.debug(f'posting to {recp}') @@ -571,7 +569,6 @@ class Like(BaseActivity): # Update the meta counter if the object is published by the server DB.outbox.update_one({'activity.object.id': obj.id}, { '$inc': {'meta.count_like': 1}, - '$addToSet': {'meta.col_likes': self.to_dict(embed=True, embed_object_id_only=True)}, }) # XXX(tsileo): notification?? @@ -580,7 +577,6 @@ class Like(BaseActivity): # Update the meta counter if the object is published by the server DB.outbox.update_one({'activity.object.id': obj.id}, { '$inc': {'meta.count_like': -1}, - '$pull': {'meta.col_likes': {'id': self.id}}, }) def _undo_should_purge_cache(self) -> bool: @@ -592,7 +588,6 @@ class Like(BaseActivity): # Unlikely, but an actor can like it's own post DB.outbox.update_one({'activity.object.id': obj.id}, { '$inc': {'meta.count_like': 1}, - '$addToSet': {'meta.col_likes': self.to_dict(embed=True, embed_object_id_only=True)}, }) # Keep track of the like we just performed @@ -603,7 +598,6 @@ class Like(BaseActivity): # Unlikely, but an actor can like it's own post DB.outbox.update_one({'activity.object.id': obj.id}, { '$inc': {'meta.count_like': -1}, - '$pull': {'meta.col_likes': {'id': self.id}}, }) DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.liked': False}}) @@ -646,7 +640,6 @@ class Announce(BaseActivity): DB.outbox.update_one({'activity.object.id': obj.id}, { '$inc': {'meta.count_boost': 1}, - '$addToSet': {'meta.col_shares': self.to_dict(embed=True, embed_object_id_only=True)}, }) def _undo_inbox(self) -> None: @@ -654,7 +647,6 @@ class Announce(BaseActivity): # Update the meta counter if the object is published by the server DB.outbox.update_one({'activity.object.id': obj.id}, { '$inc': {'meta.count_boost': -1}, - '$pull': {'meta.col_shares': {'id': self.id}}, }) def _undo_should_purge_cache(self) -> bool: @@ -1079,11 +1071,12 @@ def embed_collection(total_items, first_page_id): return { "type": ActivityType.ORDERED_COLLECTION.value, "totalItems": total_items, - "first": first_page_id, + "first": f'{first_page_id}?page=first', + "id": first_page_id, } -def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, col_name=None): +def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, col_name=None, first_page=False): col_name = col_name or col.name if q is None: q = {} @@ -1127,6 +1120,9 @@ def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, if len(data) == limit: resp['first']['next'] = BASE_URL + '/' + col_name + '?cursor=' + next_page_cursor + if first_page: + return resp['first'] + return resp # If there's a cursor, then we return an OrderedCollectionPage @@ -1141,6 +1137,9 @@ def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, if len(data) == limit: resp['next'] = BASE_URL + '/' + col_name + '?cursor=' + next_page_cursor - # TODO(tsileo): implements prev with prev= + if first_page: + return resp['first'] + + # XXX(tsileo): implements prev with prev=? return resp diff --git a/app.py b/app.py index 127c0f5..3313af3 100644 --- a/app.py +++ b/app.py @@ -62,6 +62,8 @@ from utils.webfinger import get_actor_url from utils.errors import Error from utils.errors import UnexpectedActivityTypeError from utils.errors import BadActivityError +from utils.errors import NotFromOutboxError +from utils.errors import ActivityNotFoundError from typing import Dict, Any @@ -509,7 +511,7 @@ def outbox(): DB.outbox, q=q, cursor=request.args.get('cursor'), - map_func=lambda doc: clean_activity(doc['activity']), + map_func=lambda doc: activity_from_doc(doc), )) # Handle POST request @@ -557,7 +559,7 @@ def outbox_activity_replies(item_id): data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) if not data: abort(404) - obj = activitypub.parse_activity(data) + obj = activitypub.parse_activity(data['activity']) if obj.type_enum != ActivityType.CREATE: abort(404) @@ -571,8 +573,9 @@ def outbox_activity_replies(item_id): DB.inbox, q=q, cursor=request.args.get('cursor'), - map_func=lambda doc: doc['activity'], + map_func=lambda doc: doc['activity']['object'], col_name=f'outbox/{item_id}/replies', + first_page=request.args.get('page') == 'first', )) @@ -583,7 +586,7 @@ def outbox_activity_likes(item_id): data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) if not data: abort(404) - obj = activitypub.parse_activity(data) + obj = activitypub.parse_activity(data['activity']) if obj.type_enum != ActivityType.CREATE: abort(404) @@ -600,6 +603,7 @@ def outbox_activity_likes(item_id): cursor=request.args.get('cursor'), map_func=lambda doc: doc['activity'], col_name=f'outbox/{item_id}/likes', + first_page=request.args.get('page') == 'first', )) @@ -610,7 +614,7 @@ def outbox_activity_shares(item_id): data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) if not data: abort(404) - obj = activitypub.parse_activity(data) + obj = activitypub.parse_activity(data['activity']) if obj.type_enum != ActivityType.CREATE: abort(404) @@ -627,6 +631,7 @@ def outbox_activity_shares(item_id): cursor=request.args.get('cursor'), map_func=lambda doc: doc['activity'], col_name=f'outbox/{item_id}/shares', + first_page=request.args.get('page') == 'first', )) @@ -744,16 +749,26 @@ def api_user_key(): return flask_jsonify(api_key=ADMIN_API_KEY) -def _user_api_get_note(): +def _user_api_arg(key: str) -> str: + """Try to get the given key from the requests, try JSON body, form data and query arg.""" if request.is_json: - oid = request.json.get('id') + oid = request.json.get(key) else: - oid = request.args.get('id') or request.form.get('id') + oid = request.args.get(key) or request.form.get(key) if not oid: - raise ValueError('missing id') + raise ValueError(f'missing {key}') - return activitypub.parse_activity(OBJECT_SERVICE.get(oid), expected=ActivityType.NOTE) + return oid + + +def _user_api_get_note(from_outbox: bool = False): + oid = _user_api_arg('id') + note = activitypub.parse_activity(OBJECT_SERVICE.get(oid), expected=ActivityType.NOTE) + if from_outbox and not note.id.startswith(ID): + raise NotFromOutboxError(f'cannot delete {note.id}, id must be owned by the server') + + return note def _user_api_response(**kwargs): @@ -769,64 +784,50 @@ def _user_api_response(**kwargs): @api_required def api_delete(): """API endpoint to delete a Note activity.""" - note = _user_api_get_note() + note = _user_api_get_note(from_outbox=True) + delete = note.build_delete() delete.post_to_outbox() return _user_api_response(activity=delete.id) -@app.route('/api/boost') +@app.route('/api/boost', methods=['POST']) @api_required def api_boost(): - # FIXME(tsileo): ensure a Note and not a Create is given - oid = request.args.get('id') - obj = activitypub.parse_activity(OBJECT_SERVICE.get(oid)) - announce = obj.build_announce() - announce.post_to_outbox() - if request.args.get('redirect'): - return redirect(request.args.get('redirect')) - return Response( - status=201, - headers={'Microblogpub-Created-Activity': announce.id}, - ) + note = _user_api_get_note() -@app.route('/api/like') + announce = note.build_announce() + announce.post_to_outbox() + + return _user_api_response(activity=announce.id) + + +@app.route('/api/like', methods=['POST']) @api_required def api_like(): - # FIXME(tsileo): ensure a Note and not a Create is given - oid = request.args.get('id') - obj = activitypub.parse_activity(OBJECT_SERVICE.get(oid)) - if not obj: - raise ValueError(f'unkown {oid} object') - like = obj.build_like() + note = _user_api_get_note() + + like = note.build_like() like.post_to_outbox() - if request.args.get('redirect'): - return redirect(request.args.get('redirect')) - return Response( - status=201, - headers={'Microblogpub-Created-Activity': like.id}, - ) + + return _user_api_response(activity=like.id) -@app.route('/api/undo', methods=['GET', 'POST']) +@app.route('/api/undo', methods=['POST']) @api_required def api_undo(): - oid = request.args.get('id') + oid = _user_api_arg('id') doc = DB.outbox.find_one({'$or': [{'id': oid}, {'remote_id': oid}]}) - undo_id = None - if doc: - obj = activitypub.parse_activity(doc.get('activity')) - # FIXME(tsileo): detect already undo-ed and make this API call idempotent - undo = obj.build_undo() - undo.post_to_outbox() - undo_id = undo.id - if request.args.get('redirect'): - return redirect(request.args.get('redirect')) - return Response( - status=201, - headers={'Microblogpub-Created-Activity': undo_id}, - ) + if not doc: + raise ActivityNotFoundError(f'cannot found {oid}') + + obj = activitypub.parse_activity(doc.get('activity')) + # FIXME(tsileo): detect already undo-ed and make this API call idempotent + undo = obj.build_undo() + undo.post_to_outbox() + + return _user_api_response(activity=undo.id) @app.route('/stream') @@ -980,22 +981,27 @@ def api_upload(): ) -@app.route('/api/new_note') +@app.route('/api/new_note', methods=['POST']) @api_required def api_new_note(): - source = request.args.get('content') + source = _user_api_arg('content') if not source: raise ValueError('missing content') - reply = None - if request.args.get('reply'): - reply = activitypub.parse_activity(OBJECT_SERVICE.get(request.args.get('reply'))) - source = request.args.get('content') + _reply, reply = None, None + try: + _reply = _user_api_arg('reply') + except ValueError: + pass + content, tags = parse_markdown(source) to = request.args.get('to') cc = [ID+'/followers'] - if reply: + + if _reply: + reply = activitypub.parse_activity(OBJECT_SERVICE.get(_reply)) cc.append(reply.attributedTo) + for tag in tags: if tag['type'] == 'Mention': cc.append(tag['href']) @@ -1003,7 +1009,7 @@ def api_new_note(): note = activitypub.Note( cc=cc, to=[to if to else config.AS_PUBLIC], - content=content, # TODO(tsileo): handle markdown + content=content, tag=tags, source={'mediaType': 'text/markdown', 'content': source}, inReplyTo=reply.id if reply else None @@ -1011,11 +1017,8 @@ def api_new_note(): create = note.build_create() create.post_to_outbox() - return Response( - status=201, - response='OK', - headers={'Microblogpub-Created-Activity': create.id}, - ) + return _user_api_response(activity=create.id) + @app.route('/api/stream') @api_required @@ -1026,41 +1029,38 @@ def api_stream(): ) -@app.route('/api/block') +@app.route('/api/block', methods=['POST']) @api_required def api_block(): - # FIXME(tsileo): ensure it's a Person ID - actor = request.args.get('actor') - if not actor: - raise ValueError('missing actor') - if DB.outbox.find_one({'type': ActivityType.BLOCK.value, - 'activity.object': actor, - 'meta.undo': False}): - return Response(status=201) + actor = _user_api_arg('actor') + + existing = DB.outbox.find_one({ + 'type': ActivityType.BLOCK.value, + 'activity.object': actor, + 'meta.undo': False, + }) + if existing: + return _user_api_response(activity=existing['activity']['id']) block = activitypub.Block(object=actor) block.post_to_outbox() - return Response( - status=201, - headers={'Microblogpub-Created-Activity': block.id}, - ) + + return _user_api_response(activity=block.id) -@app.route('/api/follow') +@app.route('/api/follow', methods=['POST']) @api_required def api_follow(): - actor = request.args.get('actor') - if not actor: - raise ValueError('missing actor') - if DB.following.find({'remote_actor': actor}).count() > 0: - return Response(status=201) + actor = _user_api_arg('actor') + + existing = DB.following.find_one({'remote_actor': actor}) + if existing: + return _user_api_response(activity=existing['activity']['id']) follow = activitypub.Follow(object=actor) follow.post_to_outbox() - return Response( - status=201, - headers={'Microblogpub-Created-Activity': follow.id}, - ) + + return _user_api_response(activity=follow.id) @app.route('/followers') diff --git a/docker-compose-tests.yml b/docker-compose-tests.yml index a834b93..280f6c3 100644 --- a/docker-compose-tests.yml +++ b/docker-compose-tests.yml @@ -1,7 +1,7 @@ version: '3.5' services: web: - build: . + image: 'microblogpub:latest' ports: - "${WEB_PORT}:5005" links: @@ -16,7 +16,7 @@ services: - MICROBLOGPUB_DEBUG=1 celery: # image: "instance1_web" - build: . + image: 'microblogpub:latest' links: - mongo - rmq diff --git a/docker-compose.yml b/docker-compose.yml index 0218494..b7f9521 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '2' services: web: - build: . + image: 'microblogpub:latest' ports: - "${WEB_PORT}:5005" links: @@ -14,7 +14,7 @@ services: - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// - MICROBLOGPUB_MONGODB_HOST=mongo:27017 celery: - build: . + image: 'microblogpub:latest' links: - mongo - rmq diff --git a/tasks.py b/tasks.py index 7e45581..c9ce2b0 100644 --- a/tasks.py +++ b/tasks.py @@ -1,4 +1,5 @@ import os +import json import logging import random @@ -13,6 +14,7 @@ from config import KEY from config import USER_AGENT from utils.httpsig import HTTPSigAuth from utils.opengraph import fetch_og_metadata +from utils.linked_data_sig import generate_signature log = logging.getLogger(__name__) @@ -22,11 +24,14 @@ SigAuth = HTTPSigAuth(ID+'#main-key', KEY.privkey) @app.task(bind=True, max_retries=12) -def post_to_inbox(self, payload, to): +def post_to_inbox(self, payload: str, to: str) -> None: try: log.info('payload=%s', payload) + log.info('generating sig') + signed_payload = json.loads(payload) + generate_signature(signed_payload, KEY.privkey) log.info('to=%s', to) - resp = requests.post(to, data=payload, auth=SigAuth, headers={ + resp = requests.post(to, data=json.dumps(signed_payload), auth=SigAuth, headers={ 'Content-Type': HEADERS[1], 'Accept': HEADERS[1], 'User-Agent': USER_AGENT, diff --git a/tests/federation_test.py b/tests/federation_test.py index 9a6c5a1..e050afc 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -5,6 +5,9 @@ import requests from html2text import html2text from utils import activitypub_utils +from typing import Tuple +from typing import List + def resp2plaintext(resp): """Convert the body of a requests reponse to plain text in order to make basic assertions.""" @@ -17,107 +20,149 @@ class Instance(object): def __init__(self, name, host_url, docker_url=None): self.host_url = host_url self.docker_url = docker_url or host_url - self.session = requests.Session() self._create_delay = 10 - with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), f'fixtures/{name}/config/admin_api_key.key')) as f: + with open( + os.path.join(os.path.dirname(os.path.abspath(__file__)), f'fixtures/{name}/config/admin_api_key.key') + ) as f: api_key = f.read() self._auth_headers = {'Authorization': f'Bearer {api_key}'} def _do_req(self, url, headers): + """Used to parse collection.""" url = url.replace(self.docker_url, self.host_url) resp = requests.get(url, headers=headers) resp.raise_for_status() return resp.json() def _parse_collection(self, payload=None, url=None): + """Parses a collection (go through all the pages).""" return activitypub_utils.parse_collection(url=url, payload=payload, do_req=self._do_req) def ping(self): """Ensures the homepage is reachable.""" - resp = self.session.get(f'{self.host_url}/') + resp = requests.get(f'{self.host_url}/') resp.raise_for_status() assert resp.status_code == 200 def debug(self): - resp = self.session.get(f'{self.host_url}/api/debug', headers={'Accept': 'application/json'}) + """Returns the debug infos (number of items in the inbox/outbox.""" + resp = requests.get( + f'{self.host_url}/api/debug', + headers={**self._auth_headers, 'Accept': 'application/json'}, + ) resp.raise_for_status() return resp.json() - + def drop_db(self): - resp = self.session.delete(f'{self.host_url}/api/debug', headers={'Accept': 'application/json'}) + """Drops the MongoDB DB.""" + resp = requests.delete( + f'{self.host_url}/api/debug', + headers={**self._auth_headers, 'Accept': 'application/json'}, + ) resp.raise_for_status() return resp.json() - def login(self): - resp = self.session.post(f'{self.host_url}/login', data={'pass': 'hello'}) - resp.raise_for_status() - assert resp.status_code == 200 - def block(self, actor_url) -> None: + """Blocks an actor.""" # Instance1 follows instance2 - resp = self.session.get(f'{self.host_url}/api/block', params={'actor': actor_url}) + resp = requests.post( + f'{self.host_url}/api/block', + params={'actor': actor_url}, + headers=self._auth_headers, + ) assert resp.status_code == 201 # We need to wait for the Follow/Accept dance time.sleep(self._create_delay/2) - return resp.headers.get('microblogpub-created-activity') + return resp.json().get('activity') - def follow(self, instance: 'Instance') -> None: + def follow(self, instance: 'Instance') -> str: + """Follows another instance.""" # Instance1 follows instance2 - resp = self.session.get(f'{self.host_url}/api/follow', params={'actor': instance.docker_url}) + resp = requests.post( + f'{self.host_url}/api/follow', + json={'actor': instance.docker_url}, + headers=self._auth_headers, + ) assert resp.status_code == 201 # We need to wait for the Follow/Accept dance time.sleep(self._create_delay) - return resp.headers.get('microblogpub-created-activity') + return resp.json().get('activity') - def new_note(self, content, reply=None): + def new_note(self, content, reply=None) -> str: + """Creates a new note.""" params = {'content': content} if reply: params['reply'] = reply - resp = self.session.get(f'{self.host_url}/api/new_note', params=params) - assert resp.status_code == 201 - time.sleep(self._create_delay) - return resp.headers.get('microblogpub-created-activity') - - def boost(self, activity_id): - resp = self.session.get(f'{self.host_url}/api/boost', params={'id': activity_id}) - assert resp.status_code == 201 - - time.sleep(self._create_delay) - return resp.headers.get('microblogpub-created-activity') - - def like(self, activity_id): - resp = self.session.get(f'{self.host_url}/api/like', params={'id': activity_id}) - assert resp.status_code == 201 - - time.sleep(self._create_delay) - return resp.headers.get('microblogpub-created-activity') - - def delete(self, oid: str) -> None: resp = requests.post( - f'{self.host_url}/api/note/delete', - json={'id': oid}, - headers=self._auth_headers, + f'{self.host_url}/api/new_note', + json=params, + headers=self._auth_headers, ) assert resp.status_code == 201 time.sleep(self._create_delay) return resp.json().get('activity') - def undo(self, oid: str) -> None: - resp = self.session.get(f'{self.host_url}/api/undo', params={'id': oid}) + def boost(self, oid: str) -> str: + """Creates an Announce activity.""" + resp = requests.post( + f'{self.host_url}/api/boost', + json={'id': oid}, + headers=self._auth_headers, + ) + assert resp.status_code == 201 + + time.sleep(self._create_delay) + return resp.json().get('activity') + + def like(self, oid: str) -> str: + """Creates a Like activity.""" + resp = requests.post( + f'{self.host_url}/api/like', + json={'id': oid}, + headers=self._auth_headers, + ) + assert resp.status_code == 201 + + time.sleep(self._create_delay) + return resp.json().get('activity') + + def delete(self, oid: str) -> str: + """Creates a Delete activity.""" + resp = requests.post( + f'{self.host_url}/api/note/delete', + json={'id': oid}, + headers=self._auth_headers, + ) + assert resp.status_code == 201 + + time.sleep(self._create_delay) + return resp.json().get('activity') + + def undo(self, oid: str) -> str: + """Creates a Undo activity.""" + resp = requests.post( + f'{self.host_url}/api/undo', + json={'id': oid}, + headers=self._auth_headers, + ) assert resp.status_code == 201 # We need to wait for the Follow/Accept dance time.sleep(self._create_delay) - return resp.headers.get('microblogpub-created-activity') + return resp.json().get('activity') - def followers(self): - resp = self.session.get(f'{self.host_url}/followers', headers={'Accept': 'application/activity+json'}) + def followers(self) -> List[str]: + """Parses the followers collection.""" + resp = requests.get( + f'{self.host_url}/followers', + headers={'Accept': 'application/activity+json'}, + ) resp.raise_for_status() data = resp.json() @@ -125,7 +170,11 @@ class Instance(object): return self._parse_collection(payload=data) def following(self): - resp = self.session.get(f'{self.host_url}/following', headers={'Accept': 'application/activity+json'}) + """Parses the following collection.""" + resp = requests.get( + f'{self.host_url}/following', + headers={'Accept': 'application/activity+json'}, + ) resp.raise_for_status() data = resp.json() @@ -133,38 +182,50 @@ class Instance(object): return self._parse_collection(payload=data) def outbox(self): - resp = self.session.get(f'{self.host_url}/following', headers={'Accept': 'application/activity+json'}) + """Returns the instance outbox.""" + resp = requests.get( + f'{self.host_url}/following', + headers={'Accept': 'application/activity+json'}, + ) resp.raise_for_status() return resp.json() def outbox_get(self, aid): - resp = self.session.get(aid.replace(self.docker_url, self.host_url), headers={'Accept': 'application/activity+json'}) + """Fetches a specific item from the instance outbox.""" + resp = requests.get( + aid.replace(self.docker_url, self.host_url), + headers={'Accept': 'application/activity+json'}, + ) resp.raise_for_status() return resp.json() def stream_jsonfeed(self): - resp = self.session.get(f'{self.host_url}/api/stream', headers={'Accept': 'application/json'}) + """Returns the "stream"'s JSON feed.""" + resp = requests.get( + f'{self.host_url}/api/stream', + headers={**self._auth_headers, 'Accept': 'application/json'}, + ) resp.raise_for_status() return resp.json() -def _instances(): +def _instances() -> Tuple[Instance, Instance]: + """Initializes the client for the two test instances.""" instance1 = Instance('instance1', 'http://localhost:5006', 'http://instance1_web_1:5005') instance1.ping() instance2 = Instance('instance2', 'http://localhost:5007', 'http://instance2_web_1:5005') instance2.ping() - # Login - instance1.login() + # Return the DB instance1.drop_db() - instance2.login() instance2.drop_db() - + return instance1, instance2 -def test_follow(): +def test_follow() -> None: + """instance1 follows instance2.""" instance1, instance2 = _instances() # Instance1 follows instance2 instance1.follow(instance2) @@ -181,6 +242,7 @@ def test_follow(): def test_follow_unfollow(): + """instance1 follows instance2, then unfollows it.""" instance1, instance2 = _instances() # Instance1 follows instance2 follow_id = instance1.follow(instance2) @@ -210,6 +272,7 @@ def test_follow_unfollow(): def test_post_content(): + """Instances follow each other, and instance1 creates a note.""" instance1, instance2 = _instances() # Instance1 follows instance2 instance1.follow(instance2) @@ -230,6 +293,7 @@ def test_post_content(): def test_block_and_post_content(): + """Instances follow each other, instance2 blocks instance1, instance1 creates a new note.""" instance1, instance2 = _instances() # Instance1 follows instance2 instance1.follow(instance2) @@ -251,6 +315,7 @@ def test_block_and_post_content(): def test_post_content_and_delete(): + """Instances follow each other, instance1 creates a new note, then deletes it.""" instance1, instance2 = _instances() # Instance1 follows instance2 instance1.follow(instance2) @@ -280,6 +345,7 @@ def test_post_content_and_delete(): def test_post_content_and_like(): + """Instances follow each other, instance1 creates a new note, instance2 likes it.""" instance1, instance2 = _instances() # Instance1 follows instance2 instance1.follow(instance2) @@ -302,10 +368,13 @@ def test_post_content_and_like(): note = instance1.outbox_get(f'{create_id}/activity') assert 'likes' in note assert note['likes']['totalItems'] == 1 - # assert note['likes']['items'][0]['id'] == like_id + likes = instance1._parse_collection(url=note['likes']['first']) + assert len(likes) == 1 + assert likes[0]['id'] == like_id -def test_post_content_and_like_unlike(): +def test_post_content_and_like_unlike() -> None: + """Instances follow each other, instance1 creates a new note, instance2 likes it, then unlikes it.""" instance1, instance2 = _instances() # Instance1 follows instance2 instance1.follow(instance2) @@ -328,8 +397,9 @@ def test_post_content_and_like_unlike(): note = instance1.outbox_get(f'{create_id}/activity') assert 'likes' in note assert note['likes']['totalItems'] == 1 - # FIXME(tsileo): parse the collection - # assert note['likes']['items'][0]['id'] == like_id + likes = instance1._parse_collection(url=note['likes']['first']) + assert len(likes) == 1 + assert likes[0]['id'] == like_id instance2.undo(like_id) @@ -342,7 +412,8 @@ def test_post_content_and_like_unlike(): assert note['likes']['totalItems'] == 0 -def test_post_content_and_boost(): +def test_post_content_and_boost() -> None: + """Instances follow each other, instance1 creates a new note, instance2 "boost" it.""" instance1, instance2 = _instances() # Instance1 follows instance2 instance1.follow(instance2) @@ -365,11 +436,13 @@ def test_post_content_and_boost(): note = instance1.outbox_get(f'{create_id}/activity') assert 'shares' in note assert note['shares']['totalItems'] == 1 - # FIXME(tsileo): parse the collection - # assert note['shares']['items'][0]['id'] == boost_id + shares = instance1._parse_collection(url=note['shares']['first']) + assert len(shares) == 1 + assert shares[0]['id'] == boost_id -def test_post_content_and_boost_unboost(): +def test_post_content_and_boost_unboost() -> None: + """Instances follow each other, instance1 creates a new note, instance2 "boost" it, then "unboost" it.""" instance1, instance2 = _instances() # Instance1 follows instance2 instance1.follow(instance2) @@ -392,8 +465,9 @@ def test_post_content_and_boost_unboost(): note = instance1.outbox_get(f'{create_id}/activity') assert 'shares' in note assert note['shares']['totalItems'] == 1 - # FIXME(tsileo): parse the collection - # assert note['shares']['items'][0]['id'] == boost_id + shares = instance1._parse_collection(url=note['shares']['first']) + assert len(shares) == 1 + assert shares[0]['id'] == boost_id instance2.undo(boost_id) @@ -406,7 +480,8 @@ def test_post_content_and_boost_unboost(): assert note['shares']['totalItems'] == 0 -def test_post_content_and_post_reply(): +def test_post_content_and_post_reply() -> None: + """Instances follow each other, instance1 creates a new note, instance2 replies to it.""" instance1, instance2 = _instances() # Instance1 follows instance2 instance1.follow(instance2) @@ -425,7 +500,10 @@ def test_post_content_and_post_reply(): assert len(instance2_inbox_stream['items']) == 1 assert instance2_inbox_stream['items'][0]['id'] == instance1_create_id - instance2_create_id = instance2.new_note(f'hey @instance1@{instance1.docker_url}', reply=f'{instance1_create_id}/activity') + instance2_create_id = instance2.new_note( + f'hey @instance1@{instance1.docker_url}', + reply=f'{instance1_create_id}/activity', + ) instance2_debug = instance2.debug() assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity @@ -441,10 +519,13 @@ def test_post_content_and_post_reply(): instance1_note = instance1.outbox_get(f'{instance1_create_id}/activity') assert 'replies' in instance1_note assert instance1_note['replies']['totalItems'] == 1 - # TODO(tsileo): inspect the `replies` collection + replies = instance1._parse_collection(url=instance1_note['replies']['first']) + assert len(replies) == 1 + assert replies[0]['id'] == f'{instance2_create_id}/activity' -def test_post_content_and_post_reply_and_delete(): +def test_post_content_and_post_reply_and_delete() -> None: + """Instances follow each other, instance1 creates a new note, instance2 replies to it, then deletes its reply.""" instance1, instance2 = _instances() # Instance1 follows instance2 instance1.follow(instance2) @@ -463,7 +544,10 @@ def test_post_content_and_post_reply_and_delete(): assert len(instance2_inbox_stream['items']) == 1 assert instance2_inbox_stream['items'][0]['id'] == instance1_create_id - instance2_create_id = instance2.new_note(f'hey @instance1@{instance1.docker_url}', reply=f'{instance1_create_id}/activity') + instance2_create_id = instance2.new_note( + f'hey @instance1@{instance1.docker_url}', + reply=f'{instance1_create_id}/activity', + ) instance2_debug = instance2.debug() assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity diff --git a/utils/errors.py b/utils/errors.py index 5356267..7ffe744 100644 --- a/utils/errors.py +++ b/utils/errors.py @@ -18,6 +18,9 @@ class Error(Exception): return f'{self.__class__.__qualname__}({self.message!r}, payload={self.payload!r}, status_code={self.status_code})' +class NotFromOutboxError(Error): + pass + class ActivityNotFoundError(Error): status_code = 404 diff --git a/utils/key.py b/utils/key.py index 056ee36..f5a2455 100644 --- a/utils/key.py +++ b/utils/key.py @@ -25,6 +25,7 @@ def get_secret_key(name: str, new_key: Callable[[], str] = _new_key) -> str: class Key(object): + DEFAULT_KEY_SIZE = 2048 def __init__(self, user: str, domain: str, create: bool = True) -> None: user = user.replace('.', '_') domain = domain.replace('.', '_') @@ -37,7 +38,7 @@ class Key(object): else: if not create: raise Exception('must init private key first') - k = RSA.generate(4096) + k = RSA.generate(self.DEFAULT_KEY_SIZE) self.privkey_pem = k.exportKey('PEM').decode('utf-8') self.pubkey_pem = k.publickey().exportKey('PEM').decode('utf-8') with open(key_path, 'w') as f: From 8af33d866d3e244419f2d802bba0cd0f9560e9da Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 1 Jun 2018 20:59:32 +0200 Subject: [PATCH 0077/1425] Fix the Update handling --- activitypub.py | 9 ++++++--- docker-compose-dev.yml | 2 +- utils/httpsig.py | 9 +++++++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/activitypub.py b/activitypub.py index 1cd048c..88ff84b 100644 --- a/activitypub.py +++ b/activitypub.py @@ -719,12 +719,13 @@ class Update(BaseActivity): # TODO(tsileo): implements _should_purge_cache if it's a reply of a published activity (i.e. in the outbox) def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - obj = self.get_object() + print('UPDATE') + obj = self._data['object'] update_prefix = 'activity.object.' update: Dict[str, Any] = {'$set': dict(), '$unset': dict()} update['$set'][f'{update_prefix}updated'] = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' - for k, v in obj._data.items(): + for k, v in obj.items(): if k in ['id', 'type']: continue if v is None: @@ -735,7 +736,9 @@ class Update(BaseActivity): if len(update['$unset']) == 0: del(update['$unset']) - DB.outbox.update_one({'remote_id': obj.id.replace('/activity', '')}, update) + print(f'updating note from outbox {obj!r} {update}') + logger.info(f'updating note from outbox {obj!r} {update}') + DB.outbox.update_one({'activity.object.id': obj['id']}, update) # FIXME(tsileo): should send an Update (but not a partial one, to all the note's recipients # (create a new Update with the result of the update, and send it without saving it?) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index bf6b0ee..980264e 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -1,7 +1,7 @@ version: '2' services: celery: - build: . + image: microblogpub:latest links: - mongo - rabbitmq diff --git a/utils/httpsig.py b/utils/httpsig.py index f63c12c..8437784 100644 --- a/utils/httpsig.py +++ b/utils/httpsig.py @@ -5,7 +5,7 @@ Mastodon instances won't accept requests that are not signed using this scheme. """ from datetime import datetime from urllib.parse import urlparse -from typing import Any, Dict +from typing import Any, Dict, Optional import base64 import hashlib import logging @@ -31,7 +31,9 @@ def _build_signed_string(signed_headers: str, method: str, path: str, headers: A return '\n'.join(out) -def _parse_sig_header(val: str) -> Dict[str, str]: +def _parse_sig_header(val: Optional[str]) -> Optional[Dict[str, str]]: + if not val: + return None out = {} for data in val.split(','): k, v = data.split('=', 1) @@ -54,6 +56,9 @@ def _body_digest() -> str: def verify_request(actor_service) -> bool: hsig = _parse_sig_header(request.headers.get('Signature')) + if not hsig: + logger.debug('no signature in header') + return False logger.debug(f'hsig={hsig}') signed_string = _build_signed_string(hsig['headers'], request.method, request.path, request.headers, _body_digest()) _, rk = actor_service.get_public_key(hsig['keyId']) From 2befde27d57878f892f059f66d776f03dd97ecae Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 1 Jun 2018 21:54:43 +0200 Subject: [PATCH 0078/1425] Enable the CSRF check for the login page --- activitypub.py | 3 ++- app.py | 3 ++- templates/login.html | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/activitypub.py b/activitypub.py index 88ff84b..0b4eca6 100644 --- a/activitypub.py +++ b/activitypub.py @@ -293,6 +293,8 @@ class BaseActivity(object): def _should_purge_cache(self) -> bool: raise NotImplementedError + # FIXME(tsileo): _pre_process_from_inbox, _pre_post_to_outbox, allow to prevent saving, check for undo, delete, update both inbox and outbox + def process_from_inbox(self) -> None: logger.debug(f'calling main process from inbox hook for {self}') self.verify() @@ -719,7 +721,6 @@ class Update(BaseActivity): # TODO(tsileo): implements _should_purge_cache if it's a reply of a published activity (i.e. in the outbox) def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - print('UPDATE') obj = self._data['object'] update_prefix = 'activity.object.' diff --git a/app.py b/app.py index 3313af3..0391e40 100644 --- a/app.py +++ b/app.py @@ -73,7 +73,7 @@ app.secret_key = get_secret_key('flask') app.config.update( WTF_CSRF_CHECK_DEFAULT=False, ) -# csrf = CSRFProtect(app) +csrf = CSRFProtect(app) logger = logging.getLogger(__name__) @@ -287,6 +287,7 @@ def login(): devices = [doc['device'] for doc in DB.u2f.find()] u2f_enabled = True if devices else False if request.method == 'POST': + csrf.protect() pwd = request.form.get('pass') if pwd and verify_pass(pwd): if devices: diff --git a/templates/login.html b/templates/login.html index 7e3cc1c..603808c 100644 --- a/templates/login.html +++ b/templates/login.html @@ -8,6 +8,7 @@ {% if session.logged_in %}logged{% else%}not logged{%endif%}
    + {% if u2f_enabled %} From 84dec1e38684f76393660b62d5470721f0e329b8 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 2 Jun 2018 07:32:18 +0200 Subject: [PATCH 0079/1425] Fix build --- activitypub.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/activitypub.py b/activitypub.py index 0b4eca6..15a0454 100644 --- a/activitypub.py +++ b/activitypub.py @@ -293,7 +293,8 @@ class BaseActivity(object): def _should_purge_cache(self) -> bool: raise NotImplementedError - # FIXME(tsileo): _pre_process_from_inbox, _pre_post_to_outbox, allow to prevent saving, check for undo, delete, update both inbox and outbox + # FIXME(tsileo): _pre_process_from_inbox, _pre_post_to_outbox, allow to prevent saving, + # check for undo, delete, update both inbox and outbox def process_from_inbox(self) -> None: logger.debug(f'calling main process from inbox hook for {self}') From 791e55c7f5eee26792b90b94668d5f0dbbc767f9 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 2 Jun 2018 09:07:57 +0200 Subject: [PATCH 0080/1425] Add more security check/verification --- activitypub.py | 92 ++++++++++++++++++++++++++++++++++++++++++++------ app.py | 11 ++++-- 2 files changed, 90 insertions(+), 13 deletions(-) diff --git a/activitypub.py b/activitypub.py index 15a0454..316d620 100644 --- a/activitypub.py +++ b/activitypub.py @@ -10,7 +10,9 @@ from html2text import html2text from feedgen.feed import FeedGenerator from utils.actor_service import NotAnActorError -from utils.errors import BadActivityError, UnexpectedActivityTypeError +from utils.errors import BadActivityError +from utils.errors import UnexpectedActivityTypeError +from utils.errors import NotFromOutboxError from utils import activitypub_utils from config import USERNAME, BASE_URL, ID from config import CTX_AS, CTX_SECURITY, AS_PUBLIC @@ -94,18 +96,25 @@ class BaseActivity(object): ACTIVITY_TYPE: Optional[ActivityType] = None ALLOWED_OBJECT_TYPES: List[ActivityType] = [] + OBJECT_REQUIRED = False def __init__(self, **kwargs) -> None: # Ensure the class has an activity type defined if not self.ACTIVITY_TYPE: raise BadActivityError('Missing ACTIVITY_TYPE') + # XXX(tsileo): what to do about this check? # Ensure the activity has a type and a valid one - if kwargs.get('type') is not None and kwargs.pop('type') != self.ACTIVITY_TYPE.value: - raise UnexpectedActivityTypeError('Expect the type to be {}'.format(self.ACTIVITY_TYPE)) + # if kwargs.get('type') is None: + # raise BadActivityError('missing activity type') + + if kwargs.get('type') and kwargs.pop('type') != self.ACTIVITY_TYPE.value: + raise UnexpectedActivityTypeError(f'Expect the type to be {self.ACTIVITY_TYPE.value!r}') # Initialize the object - self._data: Dict[str, Any] = {'type': self.ACTIVITY_TYPE.value} + self._data: Dict[str, Any] = { + 'type': self.ACTIVITY_TYPE.value + } logger.debug(f'initializing a {self.ACTIVITY_TYPE.value} activity: {kwargs}') if 'id' in kwargs: @@ -118,6 +127,7 @@ class BaseActivity(object): actor = self._validate_person(actor) self._data['actor'] = actor else: + # FIXME(tsileo): uses a special method to set the actor as "the instance" if not self.NO_CONTEXT and self.ACTIVITY_TYPE != ActivityType.TOMBSTONE: actor = ID self._data['actor'] = actor @@ -166,7 +176,7 @@ class BaseActivity(object): # Allows an extra to (like for Accept and Follow) kwargs.pop('to', None) if len(set(kwargs.keys()) - set(allowed_keys)) > 0: - raise BadActivityError('extra data left: {}'.format(kwargs)) + raise BadActivityError(f'extra data left: {kwargs!r}') else: # Remove keys with `None` value valid_kwargs = {} @@ -183,6 +193,10 @@ class BaseActivity(object): raise NotImplementedError def verify(self) -> None: + """Verifies that the activity is valid.""" + if self.OBJECT_REQUIRED and 'object' not in self._data: + raise BadActivityError('activity must have an "object"') + try: self._verify() except NotImplementedError: @@ -275,12 +289,18 @@ class BaseActivity(object): actor_id = self._actor_id(actor) return Person(**ACTOR_SERVICE.get(actor_id)) + def _pre_post_to_outbox(self) -> None: + raise NotImplementedError + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: raise NotImplementedError def _undo_outbox(self) -> None: raise NotImplementedError + def _pre_process_from_inbox(self) -> None: + raise NotImplementedError + def _process_from_inbox(self) -> None: raise NotImplementedError @@ -293,9 +313,6 @@ class BaseActivity(object): def _should_purge_cache(self) -> bool: raise NotImplementedError - # FIXME(tsileo): _pre_process_from_inbox, _pre_post_to_outbox, allow to prevent saving, - # check for undo, delete, update both inbox and outbox - def process_from_inbox(self) -> None: logger.debug(f'calling main process from inbox hook for {self}') self.verify() @@ -313,6 +330,12 @@ class BaseActivity(object): logger.info(f'received duplicate activity {self}, dropping it') return + try: + self._pre_process_from_inbox() + logger.debug('called pre process from inbox hook') + except NotImplementedError: + logger.debug('pre process from inbox hook not implemented') + activity = self.to_dict() DB.inbox.insert_one({ 'activity': activity, @@ -333,6 +356,13 @@ class BaseActivity(object): obj_id = random_object_id() self.set_id(f'{ID}/outbox/{obj_id}', obj_id) self.verify() + + try: + self._pre_post_to_outbox() + logger.debug(f'called pre post to outbox hook') + except NotImplementedError: + logger.debug('pre post to outbox hook not implemented') + activity = self.to_dict() DB.outbox.insert_one({ 'id': obj_id, @@ -434,6 +464,7 @@ class Person(BaseActivity): class Block(BaseActivity): ACTIVITY_TYPE = ActivityType.BLOCK + OBJECT_REQUIRED = True class Collection(BaseActivity): @@ -456,6 +487,7 @@ class Image(BaseActivity): class Follow(BaseActivity): ACTIVITY_TYPE = ActivityType.FOLLOW ALLOWED_OBJECT_TYPES = [ActivityType.PERSON] + OBJECT_REQUIRED = True def _build_reply(self, reply_type: ActivityType) -> BaseActivity: if reply_type == ActivityType.ACCEPT: @@ -515,6 +547,7 @@ class Accept(BaseActivity): class Undo(BaseActivity): ACTIVITY_TYPE = ActivityType.UNDO ALLOWED_OBJECT_TYPES = [ActivityType.FOLLOW, ActivityType.LIKE, ActivityType.ANNOUNCE] + OBJECT_REQUIRED = True def _recipients(self) -> List[str]: obj = self.get_object() @@ -525,6 +558,13 @@ class Undo(BaseActivity): # TODO(tsileo): handle like and announce raise Exception('TODO') + def _pre_process_from_inbox(self) -> None: + """Ensures an Undo activity comes from the same actor as the updated activity.""" + obj = self.get_object() + actor = self.get_actor() + if actor.id != obj.get_actor().id: + raise BadActivityError(f'{actor!r} cannot update {obj!r}') + def _process_from_inbox(self) -> None: obj = self.get_object() DB.inbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) @@ -545,6 +585,12 @@ class Undo(BaseActivity): return False + def _pre_post_to_outbox(self) -> None: + """Ensures an Undo activity references an activity owned by the instance.""" + obj = self.get_object() + if not obj.id.startswith(ID): + raise NotFromOutboxError(f'object {obj["id"]} is not owned by this instance') + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: logger.debug('processing undo to outbox') logger.debug('self={}'.format(self)) @@ -563,6 +609,7 @@ class Undo(BaseActivity): class Like(BaseActivity): ACTIVITY_TYPE = ActivityType.LIKE ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] + OBJECT_REQUIRED = True def _recipients(self) -> List[str]: return [self.get_object().get_actor().id] @@ -680,6 +727,7 @@ class Announce(BaseActivity): class Delete(BaseActivity): ACTIVITY_TYPE = ActivityType.DELETE ALLOWED_OBJECT_TYPES = [ActivityType.NOTE, ActivityType.TOMBSTONE] + OBJECT_REQUIRED = True def _get_actual_object(self) -> BaseActivity: obj = self.get_object() @@ -691,6 +739,13 @@ class Delete(BaseActivity): obj = self._get_actual_object() return obj._recipients() + def _pre_process_from_inbox(self) -> None: + """Ensures a Delete activity comes from the same actor as the deleted activity.""" + obj = self._get_actual_object() + actor = self.get_actor() + if actor.id != obj.get_actor().id: + raise BadActivityError(f'{actor!r} cannot delete {obj!r}') + def _process_from_inbox(self) -> None: DB.inbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) obj = self._get_actual_object() @@ -699,6 +754,12 @@ class Delete(BaseActivity): # TODO(tsileo): also purge the cache if it's a reply of a published activity + def _pre_post_to_outbox(self) -> None: + """Ensures the Delete activity references a activity from the outbox (i.e. owned by the instance).""" + obj = self._get_actual_object() + if not obj.id.startswith(ID): + raise NotFromOutboxError(f'object {obj["id"]} is not owned by this instance') + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: DB.outbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) @@ -706,9 +767,14 @@ class Delete(BaseActivity): class Update(BaseActivity): ACTIVITY_TYPE = ActivityType.UPDATE ALLOWED_OBJECT_TYPES = [ActivityType.NOTE, ActivityType.PERSON] + OBJECT_REQUIRED = True - # TODO(tsileo): ensure the actor updating is the same as the orinial activity - # (ensuring that the Update and its object are of same origin) + def _pre_process_from_inbox(self) -> None: + """Ensures an Update activity comes from the same actor as the updated activity.""" + obj = self.get_object() + actor = self.get_actor() + if actor.id != obj.get_actor().id: + raise BadActivityError(f'{actor!r} cannot update {obj!r}') def _process_from_inbox(self): obj = self.get_object() @@ -721,6 +787,11 @@ class Update(BaseActivity): # TODO(tsileo): implements _should_purge_cache if it's a reply of a published activity (i.e. in the outbox) + def _pre_post_to_outbox(self) -> None: + obj = self.get_object() + if not obj.id.startswith(ID): + raise NotFromOutboxError(f'object {obj["id"]} is not owned by this instance') + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: obj = self._data['object'] @@ -748,6 +819,7 @@ class Update(BaseActivity): class Create(BaseActivity): ACTIVITY_TYPE = ActivityType.CREATE ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] + OBJECT_REQUIRED = True def _set_id(self, uri: str, obj_id: str) -> None: self._data['object']['id'] = uri + '/activity' diff --git a/app.py b/app.py index 0391e40..0fd10db 100644 --- a/app.py +++ b/app.py @@ -901,14 +901,19 @@ def inbox(): logger.debug(f'req_headers={request.headers}') logger.debug(f'raw_data={data}') try: - print(verify_request(ACTOR_SERVICE)) - except Exception: + if not verify_request(ACTOR_SERVICE): + raise Exception('failed to verify request') + except Exception: logger.exception('failed to verify request, trying to verify the payload by fetching the remote') try: data = OBJECT_SERVICE.get(data['id']) except Exception: logger.exception(f'failed to fetch remote id at {data["id"]}') - abort(422) + return Response( + status=422, + headers={'Content-Type': 'application/json'}, + response=json.dumps({'error': 'failed to verify request (using HTTP signatures or fetching the IRI)'}), + ) activity = activitypub.parse_activity(data) logger.debug(f'inbox activity={activity}/{data}') From ef5b32a33cdf8ef7b9bea91fe43d5522272835f0 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 3 Jun 2018 10:15:11 +0200 Subject: [PATCH 0081/1425] Add NodeInfo support --- app.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 0fd10db..de08979 100644 --- a/app.py +++ b/app.py @@ -447,8 +447,39 @@ def note_by_id(note_id): return render_template('note.html', me=ME, note=data, replies=replies) +@app.route('/nodeinfo') +def nodeinfo(): + return Response( + headers={'Content-Type': 'application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#'}, + response=json.dumps({ + 'version': '2.0', + 'software': {'name': 'microblogpub', 'version': f'Microblog.pub {VERSION}'}, + 'protocols': ['activitypub'], + 'services': {'inbound': [], 'outbound': []}, + 'openRegistrations': False, + 'usage': {'users': {'total': 1}, 'localPosts': DB.outbox.count()}, + 'metadata': { + 'sourceCode': 'https://github.com/tsileo/microblog.pub', + }, + }), + ) + + +@app.route('/.well-known/nodeinfo') +def wellknown_nodeinfo(): + return flask_jsonify( + links=[ + { + 'rel': 'http://nodeinfo.diaspora.software/ns/schema/2.0', + 'href': f'{ID}/nodeinfo', + } + + ], + ) + + @app.route('/.well-known/webfinger') -def webfinger(): +def wellknown_webfinger(): """Enable WebFinger support, required for Mastodon interopability.""" resource = request.args.get('resource') if resource not in ["acct:"+USERNAME+"@"+DOMAIN, ID]: From 7db48800a262c5ccf0a8158bb8007524db020e6c Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 3 Jun 2018 11:41:18 +0200 Subject: [PATCH 0082/1425] Tweak the UI (make it use the new user API) and bugfixes --- activitypub.py | 2 +- app.py | 51 ++++++++++++------------------------------ docker-compose-dev.yml | 4 ++-- tasks.py | 1 - templates/new.html | 4 +++- 5 files changed, 20 insertions(+), 42 deletions(-) diff --git a/activitypub.py b/activitypub.py index 316d620..d9764f2 100644 --- a/activitypub.py +++ b/activitypub.py @@ -182,7 +182,7 @@ class BaseActivity(object): valid_kwargs = {} for k, v in kwargs.items(): if v is None: - break + continue valid_kwargs[k] = v self._data.update(**valid_kwargs) diff --git a/app.py b/app.py index de08979..a9751ad 100644 --- a/app.py +++ b/app.py @@ -75,13 +75,15 @@ app.config.update( ) csrf = CSRFProtect(app) +logging.basicConfig(level=logging.DEBUG) + logger = logging.getLogger(__name__) # Hook up Flask logging with gunicorn -gunicorn_logger = logging.getLogger('gunicorn.error') -root_logger = logging.getLogger() -root_logger.handlers = gunicorn_logger.handlers -root_logger.setLevel(gunicorn_logger.level) +# gunicorn_logger = logging.getLogger('gunicorn.error') +# root_logger = logging.getLogger() +# root_logger.handlers = gunicorn_logger.handlers +# root_logger.setLevel(gunicorn_logger.level) SIG_AUTH = HTTPSigAuth(ID+'#main-key', KEY.privkey) @@ -460,6 +462,7 @@ def nodeinfo(): 'usage': {'users': {'total': 1}, 'localPosts': DB.outbox.count()}, 'metadata': { 'sourceCode': 'https://github.com/tsileo/microblog.pub', + 'nodeName': f'@{USERNAME}@{DOMAIN}', }, }), ) @@ -482,16 +485,16 @@ def wellknown_nodeinfo(): def wellknown_webfinger(): """Enable WebFinger support, required for Mastodon interopability.""" resource = request.args.get('resource') - if resource not in ["acct:"+USERNAME+"@"+DOMAIN, ID]: + if resource not in [f'acct:{USERNAME}@{DOMAIN}', ID]: abort(404) out = { - "subject": "acct:"+USERNAME+"@"+DOMAIN, + "subject": f'acct:{USERNAME}@{DOMAIN}', "aliases": [ID], "links": [ {"rel": "http://webfinger.net/rel/profile-page", "type": "text/html", "href": BASE_URL}, {"rel": "self", "type": "application/activity+json", "href": ID}, - {"rel":"http://ostatus.org/schema/1.0/subscribe","template": BASE_URL+"/authorize_follow?profile={uri}"}, + {"rel":"http://ostatus.org/schema/1.0/subscribe", "template": BASE_URL+"/authorize_follow?profile={uri}"}, ], } @@ -690,36 +693,9 @@ def admin(): ) -@app.route('/new', methods=['GET', 'POST']) +@app.route('/new', methods=['GET']) @login_required def new(): - if request.method == 'POST': - reply = None - if request.form.get('reply'): - reply = activitypub.parse_activity(OBJECT_SERVICE.get(request.form.get('reply'))) - source = request.form.get('content') - content, tags = parse_markdown(source) - to = request.form.get('to') - cc = [ID+'/followers'] - if reply: - cc.append(reply.attributedTo) - for tag in tags: - if tag['type'] == 'Mention': - cc.append(tag['href']) - - note = activitypub.Note( - cc=cc, - to=[to if to else config.AS_PUBLIC], - content=content, # TODO(tsileo): handle markdown - tag=tags, - source={'mediaType': 'text/markdown', 'content': source}, - inReplyTo=reply.id if reply else None - ) - - create = note.build_create() - print(create.to_dict()) - create.post_to_outbox() - reply_id = None content = '' if request.args.get('reply'): @@ -804,8 +780,9 @@ def _user_api_get_note(from_outbox: bool = False): def _user_api_response(**kwargs): - if request.args.get('redirect'): - return redirect(request.args.get('redirect')) + _redirect = _user_api_arg('redirect') + if _redirect: + return redirect(_redirect) resp = flask_jsonify(**kwargs) resp.status_code = 201 diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 980264e..b487dc9 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -1,4 +1,4 @@ -version: '2' +version: '3' services: celery: image: microblogpub:latest @@ -7,7 +7,7 @@ services: - rabbitmq command: 'celery worker -l info -A tasks' environment: - - MICROBLOGPUB_AMQP_BORKER=pyamqp://guest@rabbitmq// + - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rabbitmq// - MICROBLOGPUB_MONGODB_HOST=mongo:27017 mongo: image: "mongo:latest" diff --git a/tasks.py b/tasks.py index c9ce2b0..a30854a 100644 --- a/tasks.py +++ b/tasks.py @@ -19,7 +19,6 @@ from utils.linked_data_sig import generate_signature log = logging.getLogger(__name__) app = Celery('tasks', broker=os.getenv('MICROBLOGPUB_AMQP_BROKER', 'pyamqp://guest@localhost//')) -# app = Celery('tasks', broker='pyamqp://guest@rabbitmq//') SigAuth = HTTPSigAuth(ID+'#main-key', KEY.privkey) diff --git a/templates/new.html b/templates/new.html index 572eb61..7e76445 100644 --- a/templates/new.html +++ b/templates/new.html @@ -5,7 +5,9 @@
    {% include "header.html" %}
    - + + + {% if reply %}{% endif %}
    From 4d72a1b16bd8f05acbefe2d2a3338e88cda6d1fd Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 3 Jun 2018 12:02:00 +0200 Subject: [PATCH 0083/1425] make the like/unlike actions use the new user API --- sass/base_theme.scss | 7 ++++++- static/css/theme.css | 2 +- templates/utils.html | 14 ++++++++++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/sass/base_theme.scss b/sass/base_theme.scss index 8a6fba0..f8c7771 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -159,7 +159,12 @@ a:hover { margin-right:5px; border-radius:2px; } - +button.bar-item { + border: 0 +} +form.action-form { + display: inline; +} .bottom-bar .perma-item { margin-right:5px; } diff --git a/static/css/theme.css b/static/css/theme.css index 4efc7a4..a2fe8c7 100644 --- a/static/css/theme.css +++ b/static/css/theme.css @@ -1 +1 @@ -.note-container p:first-child{margin-top:0}html,body{height:100%}body{background-color:#eee;color:#111;display:flex;flex-direction:column}.base-container{flex:1 0 auto}.footer{flex-shrink:0}a,h1,h2,h3,h4,h5,h6{color:#333}a{text-decoration:none}a:hover{text-decoration:underline}.gold{color:#1d781d}#header{margin-bottom:40px}#header .title{font-size:1.2em;padding-right:15px;color:#333}#header .title:hover{text-decoration:none}#header .subtitle-username{color:#111}#header .menu{padding:20px 0 10px 0}#header .menu ul{display:inline;list-style-type:none;padding:0}#header .menu ul li{float:left;padding-right:10px;margin-bottom:10px}#header .menu a{padding:2px 7px}#header .menu a.selected{background:#1d781d;color:#eee;border-radius:2px}#header .menu a:hover{background:#1d781d;color:#eee;text-decoration:none}#container{width:90%;max-width:720px;margin:40px auto}#container #notes{margin-top:20px}.actor-box{display:block;text-decoration:none;margin-bottom:40px}.actor-box .actor-icon{width:100%;max-width:120px;border-radius:2px}.actor-box h3{margin:0}.note{display:flex;margin-bottom:70px}.note .l{color:#333}.note .h-card{flex:initial;width:50px}.note .u-photo{width:50px;border-radius:2px}.note .note-wrapper{flex:1;padding-left:15px}.note .bottom-bar{margin-top:10px}.note .img-attachment{max-width:100%;border-radius:2px}.note h3{font-size:1.1em;color:#555}.note strong{font-weight:600}.note .note-container{clear:right;padding:10px 0}.bar-item{background:#ddd;padding:5px;color:#555;margin-right:5px;border-radius:2px}.bottom-bar .perma-item{margin-right:5px}.bottom-bar a.bar-item:hover{text-decoration:none}.footer>div{width:90%;max-width:720px;margin:40px auto}.footer a,.footer a:hover,.footer a:visited{text-decoration:underline;color:#111}.summary{color:#111;font-size:1.3em;margin-top:50px;margin-bottom:70px}.summary a,.summay a:hover{color:#111;text-decoration:underline}#followers,#following,#new{margin-top:50px}#admin{margin-top:50px}textarea,input{background:#ddd;padding:10px;color:#555;border:0px;border-radius:2px}input{padding:10px}input[type=submit]{color:#1d781d;text-transform:uppercase} +.note-container p:first-child{margin-top:0}html,body{height:100%}body{background-color:#eee;color:#111;display:flex;flex-direction:column}.base-container{flex:1 0 auto}.footer{flex-shrink:0}a,h1,h2,h3,h4,h5,h6{color:#333}a{text-decoration:none}a:hover{text-decoration:underline}.gold{color:#1d781d}#header{margin-bottom:40px}#header .title{font-size:1.2em;padding-right:15px;color:#333}#header .title:hover{text-decoration:none}#header .subtitle-username{color:#111}#header .menu{padding:20px 0 10px 0}#header .menu ul{display:inline;list-style-type:none;padding:0}#header .menu ul li{float:left;padding-right:10px;margin-bottom:10px}#header .menu a{padding:2px 7px}#header .menu a.selected{background:#1d781d;color:#eee;border-radius:2px}#header .menu a:hover{background:#1d781d;color:#eee;text-decoration:none}#container{width:90%;max-width:720px;margin:40px auto}#container #notes{margin-top:20px}.actor-box{display:block;text-decoration:none;margin-bottom:40px}.actor-box .actor-icon{width:100%;max-width:120px;border-radius:2px}.actor-box h3{margin:0}.note{display:flex;margin-bottom:70px}.note .l{color:#333}.note .h-card{flex:initial;width:50px}.note .u-photo{width:50px;border-radius:2px}.note .note-wrapper{flex:1;padding-left:15px}.note .bottom-bar{margin-top:10px}.note .img-attachment{max-width:100%;border-radius:2px}.note h3{font-size:1.1em;color:#555}.note strong{font-weight:600}.note .note-container{clear:right;padding:10px 0}.bar-item{background:#ddd;padding:5px;color:#555;margin-right:5px;border-radius:2px}button.bar-item{border:0}form.action-form{display:inline}.bottom-bar .perma-item{margin-right:5px}.bottom-bar a.bar-item:hover{text-decoration:none}.footer>div{width:90%;max-width:720px;margin:40px auto}.footer a,.footer a:hover,.footer a:visited{text-decoration:underline;color:#111}.summary{color:#111;font-size:1.3em;margin-top:50px;margin-bottom:70px}.summary a,.summay a:hover{color:#111;text-decoration:underline}#followers,#following,#new{margin-top:50px}#admin{margin-top:50px}textarea,input{background:#ddd;padding:10px;color:#555;border:0px;border-radius:2px}input{padding:10px}input[type=submit]{color:#1d781d;text-transform:uppercase} diff --git a/templates/utils.html b/templates/utils.html index 5808374..a8255d9 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -78,9 +78,19 @@ {% endif %} {% if item.meta.liked %} -unlike + + + + + + {% else %} -like +
    + + + + +
    {% endif %} {% endif %} From 8b166633c97107867bbe8f5be4d5c78d245e0293 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 3 Jun 2018 12:15:30 +0200 Subject: [PATCH 0084/1425] make the boost/unboost actions use the new user API --- templates/utils.html | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/templates/utils.html b/templates/utils.html index a8255d9..33dc7e1 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -62,19 +62,32 @@
    {% if perma %}{{ item.activity.object.published | format_time }} {% endif %} permalink + {% if item.meta.count_reply %}{{ item.meta.count_reply }} replies{% endif %} {% if item.meta.count_boost %}{{ item.meta.count_boost }} boosts{% endif %} {% if item.meta.count_like %}{{ item.meta.count_like }} likes{% endif %} + {% if ui %} + {% set aid = item.activity.object.id | quote_plus %} reply {% set redir = request.path + "#activity-" + item['_id'].__str__() %} {% if item.meta.boosted %} -unboost +
    + + + + +
    {% else %} -boost +
    + + + + +
    {% endif %} {% if item.meta.liked %} From 526725fc248d42f95aba2de57d78b6074f4b9f7d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 3 Jun 2018 12:50:51 +0200 Subject: [PATCH 0085/1425] Bugfix --- app.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index a9751ad..aab54f9 100644 --- a/app.py +++ b/app.py @@ -757,7 +757,7 @@ def api_user_key(): return flask_jsonify(api_key=ADMIN_API_KEY) -def _user_api_arg(key: str) -> str: +def _user_api_arg(key: str, **kwargs: Dict[str, Any]) -> str: """Try to get the given key from the requests, try JSON body, form data and query arg.""" if request.is_json: oid = request.json.get(key) @@ -765,6 +765,9 @@ def _user_api_arg(key: str) -> str: oid = request.args.get(key) or request.form.get(key) if not oid: + if 'default' in kwargs: + return kwargs.get('default') + raise ValueError(f'missing {key}') return oid @@ -780,7 +783,7 @@ def _user_api_get_note(from_outbox: bool = False): def _user_api_response(**kwargs): - _redirect = _user_api_arg('redirect') + _redirect = _user_api_arg('redirect', default=None) if _redirect: return redirect(_redirect) From 31300e20d7021f537e26a9ddec31773c773a96ec Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 3 Jun 2018 12:51:57 +0200 Subject: [PATCH 0086/1425] Fix logging --- app.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/app.py b/app.py index aab54f9..ebc775a 100644 --- a/app.py +++ b/app.py @@ -75,15 +75,13 @@ app.config.update( ) csrf = CSRFProtect(app) -logging.basicConfig(level=logging.DEBUG) - logger = logging.getLogger(__name__) # Hook up Flask logging with gunicorn -# gunicorn_logger = logging.getLogger('gunicorn.error') -# root_logger = logging.getLogger() -# root_logger.handlers = gunicorn_logger.handlers -# root_logger.setLevel(gunicorn_logger.level) +gunicorn_logger = logging.getLogger('gunicorn.error') +root_logger = logging.getLogger() +root_logger.handlers = gunicorn_logger.handlers +root_logger.setLevel(gunicorn_logger.level) SIG_AUTH = HTTPSigAuth(ID+'#main-key', KEY.privkey) From 786816f0a2ec13b39ca940787ffac4372f22e06d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 3 Jun 2018 14:34:04 +0200 Subject: [PATCH 0087/1425] Tweak replies management, improve tombstone support --- activitypub.py | 15 ++++++++++++--- app.py | 19 +++++++++++++++---- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/activitypub.py b/activitypub.py index d9764f2..0b4fa0b 100644 --- a/activitypub.py +++ b/activitypub.py @@ -866,21 +866,21 @@ class Create(BaseActivity): 'meta.count_reply': 1, 'meta.count_direct_reply': direct_reply, }, + '$addToSet': {'meta.thread_children': obj.id}, }): DB.outbox.update_one({'activity.object.id': reply.id}, { '$inc': { 'meta.count_reply': 1, 'meta.count_direct_reply': direct_reply, }, + '$addToSet': {'meta.thread_children': obj.id}, }) direct_reply = 0 reply_id = reply.id reply = reply.get_local_reply() logger.debug(f'next_reply={reply}') - if reply: - # Only append to threads if it's not the root - threads.append(reply_id) + threads.append(reply_id) if reply_id: if not DB.inbox.find_one_and_update({'activity.object.id': obj.id}, { @@ -1018,6 +1018,15 @@ class Note(BaseActivity): return Delete(object=Tombstone(id=self.id).to_dict(embed=True)) + def get_tombstone(self, deleted: Optional[str]) -> BaseActivity: + return Tombstone( + id=self.id, + published=self.published, + deleted=deleted, + updated=updated, + ) + + _ACTIVITY_TYPE_TO_CLS = { ActivityType.IMAGE: Image, ActivityType.PERSON: Person, diff --git a/app.py b/app.py index ebc775a..60e786d 100644 --- a/app.py +++ b/app.py @@ -409,9 +409,11 @@ def index(): @app.route('/note/') def note_by_id(note_id): - data = DB.outbox.find_one({'id': note_id, 'meta.deleted': False}) + data = DB.outbox.find_one({'id': note_id}) if not data: - return Response(status=404) + abort(404) + if data['meta'].get('deleted', False): + abort(410) replies = list(DB.inbox.find({ 'type': 'Create', @@ -570,12 +572,18 @@ def outbox(): @app.route('/outbox/') def outbox_detail(item_id): - doc = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + doc = DB.outbox.find_one({'id': item_id}) + if doc['meta'].get('deleted', False): + obj = activitypub.parse_activity(doc['activity']) + resp = jsonify(**obj.get_object().get_tombstone()) + resp.status_code = 410 + return resp return jsonify(**activity_from_doc(doc)) @app.route('/outbox//activity') def outbox_activity(item_id): + # TODO(tsileo): handle Tombstone data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) if not data: abort(404) @@ -587,6 +595,7 @@ def outbox_activity(item_id): @app.route('/outbox//replies') def outbox_activity_replies(item_id): + # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) @@ -614,6 +623,7 @@ def outbox_activity_replies(item_id): @app.route('/outbox//likes') def outbox_activity_likes(item_id): + # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) @@ -642,6 +652,7 @@ def outbox_activity_likes(item_id): @app.route('/outbox//shares') def outbox_activity_shares(item_id): + # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) @@ -755,7 +766,7 @@ def api_user_key(): return flask_jsonify(api_key=ADMIN_API_KEY) -def _user_api_arg(key: str, **kwargs: Dict[str, Any]) -> str: +def _user_api_arg(key: str, **kwargs): """Try to get the given key from the requests, try JSON body, form data and query arg.""" if request.is_json: oid = request.json.get(key) From ca6cbc16757b73cc7d129ba82a34a999ce32325b Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 3 Jun 2018 14:55:59 +0200 Subject: [PATCH 0088/1425] Fix flake8 warning --- activitypub.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/activitypub.py b/activitypub.py index 0b4fa0b..3646c0a 100644 --- a/activitypub.py +++ b/activitypub.py @@ -1017,13 +1017,12 @@ class Note(BaseActivity): def build_delete(self) -> BaseActivity: return Delete(object=Tombstone(id=self.id).to_dict(embed=True)) - def get_tombstone(self, deleted: Optional[str]) -> BaseActivity: return Tombstone( id=self.id, published=self.published, deleted=deleted, - updated=updated, + updated=deleted, ) From b8adff6a5ff5f4085da91496a396a0f27e931805 Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Sun, 3 Jun 2018 10:25:21 -0400 Subject: [PATCH 0089/1425] Add a couple of more hints about getting started --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b8fe115..2458cd2 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,9 @@ Activities are verified using HTTP Signatures or by fetching the content on the ### Installation ```shell -$ git clone +$ git clone https://github.com/tsileo/microblog.pub +$ cd microblog.pub +$ pip install -r requirements.txt $ make css $ cp -r config/me.sample.yml config/me.yml ``` From f960e15cbdd6b2a8acc6cb6d2f3eb30c59fa0f91 Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Sun, 3 Jun 2018 10:27:35 -0400 Subject: [PATCH 0090/1425] Allow config to use different python executable --- Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6221cdd..925bc09 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,10 @@ +PYTHON=python + css: - python -c "import sass; sass.compile(dirname=('sass', 'static/css'), output_style='compressed')" + $(PYTHON) -c "import sass; sass.compile(dirname=('sass', 'static/css'), output_style='compressed')" password: - python -c "import bcrypt; from getpass import getpass; print(bcrypt.hashpw(getpass().encode('utf-8'), bcrypt.gensalt()).decode('utf-8'))" + $(PYTHON) -c "import bcrypt; from getpass import getpass; print(bcrypt.hashpw(getpass().encode('utf-8'), bcrypt.gensalt()).decode('utf-8'))" docker: mypy . --ignore-missing-imports From f88ba2b959cf8dded1ad4426476dc513af98521c Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Sun, 3 Jun 2018 10:39:46 -0400 Subject: [PATCH 0091/1425] Need to add password to me.yml --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index b8fe115..54b49f2 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,20 @@ $ cp -r config/me.sample.yml config/me.yml ```shell $ make password +Password: +$2b$12$iW497g... +``` + +Edit `config/me.yml` to add the above-generated password, like so: + +``` +username: 'username' +name: 'You Name' +icon_url: 'https://you-avatar-url' +domain: 'your-domain.tld' +summary: 'your summary' +https: true +pass: $2b$12$iW497g... ``` ### Deployment From 6f7f2ae91c4deb1260dcd2db15644b855d248d02 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 3 Jun 2018 18:23:44 +0200 Subject: [PATCH 0092/1425] Add a note about the admin API key --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 6405ece..8be6c3d 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,10 @@ $ docker-compose -f docker-compose-dev.yml up -d $ MICROBLOGPUB_DEBUG=1 FLASK_APP=app.py flask run -p 5005 --with-threads ``` +## API + +Your admin API key can be found at `config/admin_api_key.key`. + ## ActivityPub API ### GET / From de2959f75467ac13266f744ff1f6cc4b305505db Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Sun, 3 Jun 2018 14:55:46 -0400 Subject: [PATCH 0093/1425] Add note on docker-compose; FLASK_DEBUG=1; typo --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6405ece..d860a18 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ Edit `config/me.yml` to add the above-generated password, like so: ``` username: 'username' -name: 'You Name' +name: 'Your Name' icon_url: 'https://you-avatar-url' domain: 'your-domain.tld' summary: 'your summary' @@ -85,6 +85,8 @@ pass: $2b$12$iW497g... ### Deployment +Note: some of the docker yml files use version 3 of [docker-compose](https://docs.docker.com/compose/install/). + ```shell $ docker-compose up -d ``` @@ -100,7 +102,7 @@ $ pip install -r requirements.txt # Start the Celery worker, RabbitMQ and MongoDB $ docker-compose -f docker-compose-dev.yml up -d # Run the server locally -$ MICROBLOGPUB_DEBUG=1 FLASK_APP=app.py flask run -p 5005 --with-threads +$ FLASK_DEBUG=1 MICROBLOGPUB_DEBUG=1 FLASK_APP=app.py flask run -p 5005 --with-threads ``` ## ActivityPub API From 63b2d2870ab8b50616ba755b1c4e3e95fc17c41b Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 3 Jun 2018 21:28:06 +0200 Subject: [PATCH 0094/1425] Improve the replies/thread display --- activitypub.py | 3 ++ app.py | 102 ++++++++++++++++++++++++++-------------- sass/base_theme.scss | 5 +- static/css/theme.css | 2 +- templates/new.html | 6 +++ templates/note.html | 6 +-- templates/utils.html | 12 ++++- utils/content_helper.py | 3 +- 8 files changed, 96 insertions(+), 43 deletions(-) diff --git a/activitypub.py b/activitypub.py index 3646c0a..bb1ad48 100644 --- a/activitypub.py +++ b/activitypub.py @@ -958,12 +958,15 @@ class Note(BaseActivity): 'meta.count_reply': -1, 'meta.count_direct_reply': direct_reply, }, + '$pull': {'meta.thread_children': self.id}, + }): DB.outbox.update_one({'activity.object.id': reply.id}, { '$inc': { 'meta.count_reply': 1, 'meta.count_direct_reply': direct_reply, }, + '$pull': {'meta.thread_children': self.id}, }) direct_reply = 0 diff --git a/app.py b/app.py index 60e786d..b215a00 100644 --- a/app.py +++ b/app.py @@ -407,6 +407,50 @@ def index(): ) +def _build_thread(data, include_children=True): + data['_requested'] = True + root_id = data['meta'].get('thread_root_parent', data['activity']['object']['id']) + + thread_ids = data['meta'].get('thread_parents', []) + if include_children: + thread_ids.extend(data['meta'].get('thread_children', [])) + + query = { + 'activity.object.id': {'$in': thread_ids}, + 'type': 'Create', + 'meta.deleted': False, # TODO(tsileo): handle Tombstone instead of filtering them + } + # Fetch the root replies, and the children + replies = [data] + list(DB.inbox.find(query)) + list(DB.outbox.find(query)) + + # Index all the IDs in order to build a tree + idx = {} + for rep in replies: + rep_id = rep['activity']['object']['id'] + idx[rep_id] = rep.copy() + idx[rep_id]['_nodes'] = [] + + # Build the tree + for rep in replies: + rep_id = rep['activity']['object']['id'] + if rep_id == root_id: + continue + reply_of = rep['activity']['object']['inReplyTo'] + idx[reply_of]['_nodes'].append(rep) + + # Flatten the tree + thread = [] + def _flatten(node, level=0): + node['_level'] = level + thread.append(node) + + for snode in sorted(idx[node['activity']['object']['id']]['_nodes'], key=lambda d: d['activity']['object']['published']): + _flatten(snode, level=level+1) + _flatten(idx[root_id]) + + return thread + + @app.route('/note/') def note_by_id(note_id): data = DB.outbox.find_one({'id': note_id}) @@ -414,39 +458,8 @@ def note_by_id(note_id): abort(404) if data['meta'].get('deleted', False): abort(410) - - replies = list(DB.inbox.find({ - 'type': 'Create', - 'activity.object.inReplyTo': data['activity']['object']['id'], - 'meta.deleted': False, - })) - - # Check for "replies of replies" - others = [] - for rep in replies: - for rep_reply in rep.get('meta', {}).get('replies', []): - others.append(rep_reply['id']) - - if others: - # Fetch the latest versions of the "replies of replies" - replies2 = list(DB.inbox.find({ - 'activity.id': {'$in': others}, - })) - - replies.extend(replies2) - - replies2 = list(DB.outbox.find({ - 'activity.id': {'$in': others}, - })) - - replies.extend(replies2) - - - # Re-sort everything - replies = sorted(replies, key=lambda o: o['activity']['object']['published']) - - - return render_template('note.html', me=ME, note=data, replies=replies) + thread = _build_thread(data) + return render_template('note.html', me=ME, thread=thread, note=data) @app.route('/nodeinfo') @@ -707,14 +720,33 @@ def admin(): def new(): reply_id = None content = '' + thread = [] if request.args.get('reply'): - reply = activitypub.parse_activity(OBJECT_SERVICE.get(request.args.get('reply'))) + data = DB.inbox.find_one({'activity.object.id': request.args.get('reply')}) + if not data: + data = DB.outbox.find_one({'activity.object.id': request.args.get('reply')}) + if not data: + abort(400) + + reply = activitypub.parse_activity(data['activity']) reply_id = reply.id + if reply.type_enum == ActivityType.CREATE: + reply_id = reply.get_object().id actor = reply.get_actor() domain = urlparse(actor.id).netloc + # FIXME(tsileo): if reply of reply, fetch all participants content = f'@{actor.preferredUsername}@{domain} ' + thread = _build_thread( + data, + include_children=False, + ) - return render_template('new.html', reply=reply_id, content=content) + return render_template( + 'new.html', + reply=reply_id, + content=content, + thread=thread, + ) @app.route('/notifications') diff --git a/sass/base_theme.scss b/sass/base_theme.scss index f8c7771..eba792b 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -165,8 +165,11 @@ button.bar-item { form.action-form { display: inline; } +.perma { + font-size: 1.25em; +} .bottom-bar .perma-item { - margin-right:5px; + margin-right: 5px; } .bottom-bar a.bar-item:hover { text-decoration: none; diff --git a/static/css/theme.css b/static/css/theme.css index a2fe8c7..6b25941 100644 --- a/static/css/theme.css +++ b/static/css/theme.css @@ -1 +1 @@ -.note-container p:first-child{margin-top:0}html,body{height:100%}body{background-color:#eee;color:#111;display:flex;flex-direction:column}.base-container{flex:1 0 auto}.footer{flex-shrink:0}a,h1,h2,h3,h4,h5,h6{color:#333}a{text-decoration:none}a:hover{text-decoration:underline}.gold{color:#1d781d}#header{margin-bottom:40px}#header .title{font-size:1.2em;padding-right:15px;color:#333}#header .title:hover{text-decoration:none}#header .subtitle-username{color:#111}#header .menu{padding:20px 0 10px 0}#header .menu ul{display:inline;list-style-type:none;padding:0}#header .menu ul li{float:left;padding-right:10px;margin-bottom:10px}#header .menu a{padding:2px 7px}#header .menu a.selected{background:#1d781d;color:#eee;border-radius:2px}#header .menu a:hover{background:#1d781d;color:#eee;text-decoration:none}#container{width:90%;max-width:720px;margin:40px auto}#container #notes{margin-top:20px}.actor-box{display:block;text-decoration:none;margin-bottom:40px}.actor-box .actor-icon{width:100%;max-width:120px;border-radius:2px}.actor-box h3{margin:0}.note{display:flex;margin-bottom:70px}.note .l{color:#333}.note .h-card{flex:initial;width:50px}.note .u-photo{width:50px;border-radius:2px}.note .note-wrapper{flex:1;padding-left:15px}.note .bottom-bar{margin-top:10px}.note .img-attachment{max-width:100%;border-radius:2px}.note h3{font-size:1.1em;color:#555}.note strong{font-weight:600}.note .note-container{clear:right;padding:10px 0}.bar-item{background:#ddd;padding:5px;color:#555;margin-right:5px;border-radius:2px}button.bar-item{border:0}form.action-form{display:inline}.bottom-bar .perma-item{margin-right:5px}.bottom-bar a.bar-item:hover{text-decoration:none}.footer>div{width:90%;max-width:720px;margin:40px auto}.footer a,.footer a:hover,.footer a:visited{text-decoration:underline;color:#111}.summary{color:#111;font-size:1.3em;margin-top:50px;margin-bottom:70px}.summary a,.summay a:hover{color:#111;text-decoration:underline}#followers,#following,#new{margin-top:50px}#admin{margin-top:50px}textarea,input{background:#ddd;padding:10px;color:#555;border:0px;border-radius:2px}input{padding:10px}input[type=submit]{color:#1d781d;text-transform:uppercase} +.note-container p:first-child{margin-top:0}html,body{height:100%}body{background-color:#eee;color:#111;display:flex;flex-direction:column}.base-container{flex:1 0 auto}.footer{flex-shrink:0}a,h1,h2,h3,h4,h5,h6{color:#333}a{text-decoration:none}a:hover{text-decoration:underline}.gold{color:#1d781d}#header{margin-bottom:40px}#header .title{font-size:1.2em;padding-right:15px;color:#333}#header .title:hover{text-decoration:none}#header .subtitle-username{color:#111}#header .menu{padding:20px 0 10px 0}#header .menu ul{display:inline;list-style-type:none;padding:0}#header .menu ul li{float:left;padding-right:10px;margin-bottom:10px}#header .menu a{padding:2px 7px}#header .menu a.selected{background:#1d781d;color:#eee;border-radius:2px}#header .menu a:hover{background:#1d781d;color:#eee;text-decoration:none}#container{width:90%;max-width:720px;margin:40px auto}#container #notes{margin-top:20px}.actor-box{display:block;text-decoration:none;margin-bottom:40px}.actor-box .actor-icon{width:100%;max-width:120px;border-radius:2px}.actor-box h3{margin:0}.note{display:flex;margin-bottom:70px}.note .l{color:#333}.note .h-card{flex:initial;width:50px}.note .u-photo{width:50px;border-radius:2px}.note .note-wrapper{flex:1;padding-left:15px}.note .bottom-bar{margin-top:10px}.note .img-attachment{max-width:100%;border-radius:2px}.note h3{font-size:1.1em;color:#555}.note strong{font-weight:600}.note .note-container{clear:right;padding:10px 0}.bar-item{background:#ddd;padding:5px;color:#555;margin-right:5px;border-radius:2px}button.bar-item{border:0}form.action-form{display:inline}.perma{font-size:1.25em}.bottom-bar .perma-item{margin-right:5px}.bottom-bar a.bar-item:hover{text-decoration:none}.footer>div{width:90%;max-width:720px;margin:40px auto}.footer a,.footer a:hover,.footer a:visited{text-decoration:underline;color:#111}.summary{color:#111;font-size:1.3em;margin-top:50px;margin-bottom:70px}.summary a,.summay a:hover{color:#111;text-decoration:underline}#followers,#following,#new{margin-top:50px}#admin{margin-top:50px}textarea,input{background:#ddd;padding:10px;color:#555;border:0px;border-radius:2px}input{padding:10px}input[type=submit]{color:#1d781d;text-transform:uppercase} diff --git a/templates/new.html b/templates/new.html index 7e76445..bf658aa 100644 --- a/templates/new.html +++ b/templates/new.html @@ -5,6 +5,12 @@
    {% include "header.html" %}
    +{% if thread %} +

    Replying to {{ content }}

    +{{ utils.display_thread(thread) }} +{% else %} +

    New note

    +{% endif %}
    diff --git a/templates/note.html b/templates/note.html index 9311c43..b682238 100644 --- a/templates/note.html +++ b/templates/note.html @@ -16,9 +16,7 @@ {% block content %}
    {% include "header.html" %} -{{ utils.display_note(note, perma=True) }} -{% for reply in replies %} -{{ utils.display_note(reply, perma=False) }} -{% endfor %} +{{ thread }} +{{ utils.display_thread(thread) }}
    {% endblock %} diff --git a/templates/utils.html b/templates/utils.html index 33dc7e1..b33185c 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -67,7 +67,7 @@ {% if item.meta.count_boost %}{{ item.meta.count_boost }} boosts{% endif %} {% if item.meta.count_like %}{{ item.meta.count_like }} likes{% endif %} -{% if ui %} +{% if ui and session.logged_in %} {% set aid = item.activity.object.id | quote_plus %} reply @@ -112,3 +112,13 @@
    {%- endmacro %} + +{% macro display_thread(thread) -%} +{% for reply in thread %} +{% if reply._requested %} +{{ display_note(reply, perma=True, ui=False) }} +{% else %} +{{ display_note(reply, perma=False, ui=True) }} +{% endif %} +{% endfor %} +{% endmacro -%} diff --git a/utils/content_helper.py b/utils/content_helper.py index 18fabf4..b254e2b 100644 --- a/utils/content_helper.py +++ b/utils/content_helper.py @@ -40,8 +40,9 @@ def mentionify(content: str) -> Tuple[str, List[Dict[str, str]]]: _, username, domain = mention.split('@') actor_url = get_actor_url(mention) p = ACTOR_SERVICE.get(actor_url) + print(p) tags.append(dict(type='Mention', href=p['id'], name=mention)) - link = f'@{username}' + link = f'@{username}' content = content.replace(mention, link) return content, tags From a160a95e820e36f2b71475d71b4d987db7b21b5c Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 3 Jun 2018 22:44:19 +0200 Subject: [PATCH 0095/1425] Fixes replies, and hide them on the homepage --- activitypub.py | 8 ++++++++ app.py | 11 ++++++++--- templates/note.html | 1 - 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/activitypub.py b/activitypub.py index bb1ad48..1b2aed2 100644 --- a/activitypub.py +++ b/activitypub.py @@ -246,6 +246,9 @@ class BaseActivity(object): raise ValueError('Invalid actor') return actor['id'] + def reset_object_cache(self) -> None: + self.__obj = None + def get_object(self) -> 'BaseActivity': if self.__obj: return self.__obj @@ -824,6 +827,7 @@ class Create(BaseActivity): def _set_id(self, uri: str, obj_id: str) -> None: self._data['object']['id'] = uri + '/activity' self._data['object']['url'] = ID + '/' + self.get_object().type.lower() + '/' + obj_id + self.reset_object_cache() def _init(self, **kwargs): obj = self.get_object() @@ -857,6 +861,8 @@ class Create(BaseActivity): threads = [] reply = obj.get_local_reply() + print(f'initial_reply={reply}') + print(f'{obj}') logger.debug(f'initial_reply={reply}') reply_id = None direct_reply = 1 @@ -881,6 +887,8 @@ class Create(BaseActivity): reply = reply.get_local_reply() logger.debug(f'next_reply={reply}') threads.append(reply_id) + # FIXME(tsileo): obj.id is None!! + print(f'reply_id={reply_id} {obj.id} {obj._data} {self.id}') if reply_id: if not DB.inbox.find_one_and_update({'activity.object.id': obj.id}, { diff --git a/app.py b/app.py index b215a00..f0b6d9a 100644 --- a/app.py +++ b/app.py @@ -78,10 +78,14 @@ csrf = CSRFProtect(app) logger = logging.getLogger(__name__) # Hook up Flask logging with gunicorn -gunicorn_logger = logging.getLogger('gunicorn.error') root_logger = logging.getLogger() -root_logger.handlers = gunicorn_logger.handlers -root_logger.setLevel(gunicorn_logger.level) +if os.getenv('FLASK_DEBUG'): + logger.setLevel(logging.DEBUG) + root_logger.setLevel(logging.DEBUG) +else: + gunicorn_logger = logging.getLogger('gunicorn.error') + root_logger.handlers = gunicorn_logger.handlers + root_logger.setLevel(gunicorn_logger.level) SIG_AUTH = HTTPSigAuth(ID+'#main-key', KEY.privkey) @@ -378,6 +382,7 @@ def index(): q = { 'type': 'Create', 'activity.object.type': 'Note', + 'activity.object.inReplyTo': None, 'meta.deleted': False, } c = request.args.get('cursor') diff --git a/templates/note.html b/templates/note.html index b682238..c6b4ac7 100644 --- a/templates/note.html +++ b/templates/note.html @@ -16,7 +16,6 @@ {% block content %}
    {% include "header.html" %} -{{ thread }} {{ utils.display_thread(thread) }}
    {% endblock %} From 8027a363598734fb1d190e65fe2595b9cf0b500f Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 3 Jun 2018 23:11:43 +0200 Subject: [PATCH 0096/1425] Start displaying the acor that liked a particular note --- activitypub.py | 8 ++++---- app.py | 12 +++++++++++- templates/note.html | 2 +- templates/utils.html | 30 +++++++++++++++++++++++++++--- 4 files changed, 43 insertions(+), 9 deletions(-) diff --git a/activitypub.py b/activitypub.py index 1b2aed2..80becfb 100644 --- a/activitypub.py +++ b/activitypub.py @@ -246,9 +246,6 @@ class BaseActivity(object): raise ValueError('Invalid actor') return actor['id'] - def reset_object_cache(self) -> None: - self.__obj = None - def get_object(self) -> 'BaseActivity': if self.__obj: return self.__obj @@ -264,9 +261,12 @@ class BaseActivity(object): p = parse_activity(obj) - self.__obj: BaseActivity = p + self.__obj: Optional[BaseActivity] = p return p + def reset_object_cache(self) -> None: + self.__obj = None + def to_dict(self, embed: bool = False, embed_object_id_only: bool = False) -> ObjectType: data = dict(self._data) if embed: diff --git a/app.py b/app.py index f0b6d9a..c7993f5 100644 --- a/app.py +++ b/app.py @@ -464,7 +464,17 @@ def note_by_id(note_id): if data['meta'].get('deleted', False): abort(410) thread = _build_thread(data) - return render_template('note.html', me=ME, thread=thread, note=data) + + + likes = list(DB.inbox.find({ + 'meta.undo': False, + 'type': ActivityType.LIKE.value, + '$or': [{'activity.object.id': data['activity']['object']['id']}, + {'activity.object': data['activity']['object']['id']}], + })) + likes = [ACTOR_SERVICE.get(doc['activity']['actor']) for doc in likes] + + return render_template('note.html', likes=likes, me=ME, thread=thread, note=data) @app.route('/nodeinfo') diff --git a/templates/note.html b/templates/note.html index c6b4ac7..0ff83ed 100644 --- a/templates/note.html +++ b/templates/note.html @@ -16,6 +16,6 @@ {% block content %}
    {% include "header.html" %} -{{ utils.display_thread(thread) }} +{{ utils.display_thread(thread, likes) }}
    {% endblock %} diff --git a/templates/utils.html b/templates/utils.html index b33185c..07fb19e 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -1,3 +1,19 @@ +{% macro display_actor_inline(follower) -%} + + +{% if not follower.icon %} + +{% else %} +{% endif %} + +
    +
    {{ follower.name or follower.preferredUsername }}
    +@{{ follower.preferredUsername }}@{{ follower.url | domain }} +
    +     +{%- endmacro %} + + {% macro display_actor(follower) -%}
    @@ -16,7 +32,7 @@ {%- endmacro %} -{% macro display_note(item, perma=False, ui=False) -%} +{% macro display_note(item, perma=False, ui=False, likes=[]) -%} {% set actor = item.activity.object.attributedTo | get_actor %}
    @@ -111,12 +127,20 @@
    +{% if likes %} +
    +

    Liked by

    {% for like in likes %} +{{ display_actor_inline(like) }} +{% endfor %} +
    +{% endif %} + {%- endmacro %} -{% macro display_thread(thread) -%} +{% macro display_thread(thread, likes=[]) -%} {% for reply in thread %} {% if reply._requested %} -{{ display_note(reply, perma=True, ui=False) }} +{{ display_note(reply, perma=True, ui=False, likes=likes) }} {% else %} {{ display_note(reply, perma=False, ui=True) }} {% endif %} From 8e645c619010c2c87060604b592bd3cff04e1835 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 3 Jun 2018 23:36:16 +0200 Subject: [PATCH 0097/1425] Also display boost/announce actors --- app.py | 10 +++++++++- templates/note.html | 2 +- templates/utils.html | 33 +++++++++++++++++++++++++-------- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/app.py b/app.py index c7993f5..e9a0bf5 100644 --- a/app.py +++ b/app.py @@ -474,7 +474,15 @@ def note_by_id(note_id): })) likes = [ACTOR_SERVICE.get(doc['activity']['actor']) for doc in likes] - return render_template('note.html', likes=likes, me=ME, thread=thread, note=data) + shares = list(DB.inbox.find({ + 'meta.undo': False, + 'type': ActivityType.ANNOUNCE.value, + '$or': [{'activity.object.id': data['activity']['object']['id']}, + {'activity.object': data['activity']['object']['id']}], + })) + shares = [ACTOR_SERVICE.get(doc['activity']['actor']) for doc in shares] + + return render_template('note.html', likes=likes, shares=shares, me=ME, thread=thread, note=data) @app.route('/nodeinfo') diff --git a/templates/note.html b/templates/note.html index 0ff83ed..8599778 100644 --- a/templates/note.html +++ b/templates/note.html @@ -16,6 +16,6 @@ {% block content %}
    {% include "header.html" %} -{{ utils.display_thread(thread, likes) }} +{{ utils.display_thread(thread, likes=likes, shares=shares) }}
    {% endblock %} diff --git a/templates/utils.html b/templates/utils.html index 07fb19e..2535e9c 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -32,7 +32,7 @@ {%- endmacro %} -{% macro display_note(item, perma=False, ui=False, likes=[]) -%} +{% macro display_note(item, perma=False, ui=False, likes=[], shares=[]) -%} {% set actor = item.activity.object.attributedTo | get_actor %}
    @@ -76,12 +76,14 @@ {% endif %}
    -{% if perma %}{{ item.activity.object.published | format_time }} {% endif %} +{% if perma %}{{ item.activity.object.published | format_time }} +{% else %} permalink {% if item.meta.count_reply %}{{ item.meta.count_reply }} replies{% endif %} {% if item.meta.count_boost %}{{ item.meta.count_boost }} boosts{% endif %} {% if item.meta.count_like %}{{ item.meta.count_like }} likes{% endif %} +{% endif %} {% if ui and session.logged_in %} @@ -124,23 +126,38 @@ {% endif %}
    -
    -
    +{% if likes or shares %} +
    {% if likes %} -
    -

    Liked by

    {% for like in likes %} +
    +

    {{ item.meta.count_like }} likes

    {% for like in likes %} {{ display_actor_inline(like) }} {% endfor %}
    {% endif %} +{% if shares %} +
    +

    {{ item.meta.count_boost }} boosts

    {% for boost in shares %} +{{ display_actor_inline(boost) }} +{% endfor %} +
    +{% endif %} +
    +{% endif %} + + + +
    + +
    {%- endmacro %} -{% macro display_thread(thread, likes=[]) -%} +{% macro display_thread(thread, likes=[], shares=[]) -%} {% for reply in thread %} {% if reply._requested %} -{{ display_note(reply, perma=True, ui=False, likes=likes) }} +{{ display_note(reply, perma=True, ui=False, likes=likes, shares=shares) }} {% else %} {{ display_note(reply, perma=False, ui=True) }} {% endif %} From 362e9c660ff42c5e26a162bd14decdbc29fa4152 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 4 Jun 2018 17:59:38 +0200 Subject: [PATCH 0098/1425] Re-enable the remote follow --- app.py | 46 +++++++++++++++++++++++++++++++----- templates/header.html | 4 ++-- templates/remote_follow.html | 3 ++- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/app.py b/app.py index e9a0bf5..7c2fe81 100644 --- a/app.py +++ b/app.py @@ -211,9 +211,9 @@ def login_required(f): def _api_required(): if session.get('logged_in'): - #if request.method not in ['GET', 'HEAD']: - # # If a standard API request is made with a "login session", it must havw a CSRF token - # csrf.protect() + if request.method not in ['GET', 'HEAD']: + # If a standard API request is made with a "login session", it must havw a CSRF token + csrf.protect() return # Token verification @@ -325,12 +325,12 @@ def login(): @app.route('/remote_follow', methods=['GET', 'POST']) -@login_required def remote_follow(): if request.method == 'GET': return render_template('remote_follow.html') - return redirect(get_remote_follow_template('@'+request.form.get('profile')).format(uri=ID)) + csrf.protect() + return redirect(get_remote_follow_template('@'+request.form.get('profile')).format(uri=f'{USERNAME}@{DOMAIN}')) @app.route('/authorize_follow', methods=['GET', 'POST']) @@ -373,7 +373,6 @@ def u2f_register(): @app.route('/') def index(): - print(request.headers.get('Accept')) if is_api_request(): return jsonify(**ME) @@ -412,6 +411,41 @@ def index(): ) +@app.route('/with_replies') +def with_replies(): + limit = 50 + q = { + 'type': 'Create', + 'activity.object.type': 'Note', + 'meta.deleted': False, + } + c = request.args.get('cursor') + if c: + q['_id'] = {'$lt': ObjectId(c)} + + outbox_data = list(DB.outbox.find({'$or': [q, {'type': 'Announce', 'meta.undo': False}]}, limit=limit).sort('_id', -1)) + cursor = None + if outbox_data and len(outbox_data) == limit: + cursor = str(outbox_data[-1]['_id']) + + for data in outbox_data: + if data['type'] == 'Announce': + print(data) + if data['activity']['object'].startswith('http'): + data['ref'] = {'activity': {'object': OBJECT_SERVICE.get(data['activity']['object'])}, 'meta': {}} + + + return render_template( + 'index.html', + me=ME, + notes=DB.inbox.find({'type': 'Create', 'activity.object.type': 'Note', 'meta.deleted': False}).count(), + followers=DB.followers.count(), + following=DB.following.count(), + outbox_data=outbox_data, + cursor=cursor, + ) + + def _build_thread(data, include_children=True): data['_requested'] = True root_id = data['meta'].get('thread_root_parent', data['activity']['object']['id']) diff --git a/templates/header.html b/templates/header.html index faa8b63..9983dca 100644 --- a/templates/header.html +++ b/templates/header.html @@ -11,6 +11,7 @@
    diff --git a/templates/remote_follow.html b/templates/remote_follow.html index 8b54d49..413c0d1 100644 --- a/templates/remote_follow.html +++ b/templates/remote_follow.html @@ -5,9 +5,10 @@ {% block content %}
    {% include "header.html" %} -

    You're about to follow me

    +

    You're about to follow me \o/

    + From b4d44294e22153f9f9e4ab3294eb51fe4040470d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 4 Jun 2018 18:01:36 +0200 Subject: [PATCH 0099/1425] Fix CSRF for the remote follow (admin side) --- templates/authorize_remote_follow.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/templates/authorize_remote_follow.html b/templates/authorize_remote_follow.html index 78b7ef0..e0fdc8d 100644 --- a/templates/authorize_remote_follow.html +++ b/templates/authorize_remote_follow.html @@ -7,8 +7,9 @@

    You're about to follow {{ profile}}

    - - + + +
    From c17a9a5a0c49f34eb57886fa9cc5e220c71df1f1 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 4 Jun 2018 18:06:04 +0200 Subject: [PATCH 0100/1425] Dedup the cc of new note before storage --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 7c2fe81..f0677e1 100644 --- a/app.py +++ b/app.py @@ -1122,7 +1122,7 @@ def api_new_note(): cc.append(tag['href']) note = activitypub.Note( - cc=cc, + cc=list(set(cc)), to=[to if to else config.AS_PUBLIC], content=content, tag=tags, From d2e62ed5b66659085b64dc7c220dcdf90a118f88 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 4 Jun 2018 18:53:44 +0200 Subject: [PATCH 0101/1425] Add delete and block in the UI --- app.py | 5 +++++ templates/utils.html | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/app.py b/app.py index f0677e1..be06f0e 100644 --- a/app.py +++ b/app.py @@ -142,6 +142,11 @@ def quote_plus(t): return urllib.parse.quote_plus(t) +@app.template_filter() +def is_from_outbox(t): + return t.startswith(ID) + + @app.template_filter() def clean(html): return clean_html(html) diff --git a/templates/utils.html b/templates/utils.html index 2535e9c..4d7ce89 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -125,6 +125,26 @@ {% endif %} {% endif %} + +{% if session.logged_in %} +{% if item.activity.id | is_from_outbox %} +
    + + + + +
    +{% else %} +
    + + + + +
    +{% endif %} +{% endif %} + +
    {% if likes or shares %} From b89ee21e494d8f303f5bd234dd9f981fbaf823b3 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 4 Jun 2018 19:10:04 +0200 Subject: [PATCH 0102/1425] Remove @context in embedded collection --- activitypub.py | 15 ++++++++++++--- app.py | 22 ++++++++++++++++------ 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/activitypub.py b/activitypub.py index 80becfb..dbe7601 100644 --- a/activitypub.py +++ b/activitypub.py @@ -30,6 +30,15 @@ ObjectType = Dict[str, Any] ObjectOrIDType = Union[str, ObjectType] +COLLECTION_CTX = [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "Hashtag": "as:Hashtag", + "sensitive": "as:sensitive", + } +] + class ActivityType(Enum): """Supported activity `type`.""" ANNOUNCE = 'Announce' @@ -1196,11 +1205,11 @@ def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, data = [_remove_id(doc) for doc in data] if map_func: data = [map_func(doc) for doc in data] - + # No cursor, this is the first page and we return an OrderedCollection if not cursor: resp = { - '@context': CTX_AS, + '@context': COLLECTION_CTX, 'id': f'{BASE_URL}/{col_name}', 'totalItems': total_items, 'type': ActivityType.ORDERED_COLLECTION.value, @@ -1223,7 +1232,7 @@ def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, # If there's a cursor, then we return an OrderedCollectionPage resp = { - '@context': CTX_AS, + '@context': COLLECTION_CTX, 'type': ActivityType.ORDERED_COLLECTION_PAGE.value, 'id': BASE_URL + '/' + col_name + '?cursor=' + start_cursor, 'totalItems': total_items, diff --git a/app.py b/app.py index be06f0e..227f318 100644 --- a/app.py +++ b/app.py @@ -601,9 +601,19 @@ def add_extra_collection(raw_doc: Dict[str, Any]) -> Dict[str, Any]: return raw_doc -def activity_from_doc(raw_doc: Dict[str, Any]) -> Dict[str, Any]: +def remove_context(activity: Dict[str, Any]) -> Dict[str, Any]: + if '@context' in activity: + del activity['@context'] + return activity + + +def activity_from_doc(raw_doc: Dict[str, Any], embed: bool = False) -> Dict[str, Any]: raw_doc = add_extra_collection(raw_doc) - return clean_activity(raw_doc['activity']) + activity = clean_activity(raw_doc['activity']) + if embed: + return remove_context(activity) + return activity + @app.route('/outbox', methods=['GET', 'POST']) @@ -621,7 +631,7 @@ def outbox(): DB.outbox, q=q, cursor=request.args.get('cursor'), - map_func=lambda doc: activity_from_doc(doc), + map_func=lambda doc: activity_from_doc(doc, embed=True), )) # Handle POST request @@ -719,7 +729,7 @@ def outbox_activity_likes(item_id): DB.inbox, q=q, cursor=request.args.get('cursor'), - map_func=lambda doc: doc['activity'], + map_func=lambda doc: remove_context(doc['activity']), col_name=f'outbox/{item_id}/likes', first_page=request.args.get('page') == 'first', )) @@ -748,7 +758,7 @@ def outbox_activity_shares(item_id): DB.inbox, q=q, cursor=request.args.get('cursor'), - map_func=lambda doc: doc['activity'], + map_func=lambda doc: remove_context(doc['activity']), col_name=f'outbox/{item_id}/shares', first_page=request.args.get('page') == 'first', )) @@ -1008,7 +1018,7 @@ def inbox(): DB.inbox, q={'meta.deleted': False}, cursor=request.args.get('cursor'), - map_func=lambda doc: doc['activity'], + map_func=lambda doc: remove_context(doc['activity']), )) data = request.get_json(force=True) From cc5896f52054775850cbc348969de99d4d56e548 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 4 Jun 2018 19:13:04 +0200 Subject: [PATCH 0103/1425] Fix flake8 warning --- activitypub.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/activitypub.py b/activitypub.py index dbe7601..cdb0bd1 100644 --- a/activitypub.py +++ b/activitypub.py @@ -30,15 +30,16 @@ ObjectType = Dict[str, Any] ObjectOrIDType = Union[str, ObjectType] -COLLECTION_CTX = [ +COLLECTION_CTX = [ "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", + "https://w3id.org/security/v1", { "Hashtag": "as:Hashtag", "sensitive": "as:sensitive", } ] + class ActivityType(Enum): """Supported activity `type`.""" ANNOUNCE = 'Announce' @@ -1205,7 +1206,7 @@ def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, data = [_remove_id(doc) for doc in data] if map_func: data = [map_func(doc) for doc in data] - + # No cursor, this is the first page and we return an OrderedCollection if not cursor: resp = { From 070e39bdfe9d06ec44383a3f68eea21246a20be9 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 5 Jun 2018 07:33:17 +0200 Subject: [PATCH 0104/1425] Disable IndieAuth for now --- app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 227f318..e3cd805 100644 --- a/app.py +++ b/app.py @@ -1354,9 +1354,8 @@ def indieauth_flow(): return redirect(red) -@app.route('/indieauth', methods=['GET', 'POST']) +# @app.route('/indieauth', methods=['GET', 'POST']) def indieauth_endpoint(): - session['logged_in'] = True if request.method == 'GET': if not session.get('logged_in'): return redirect(url_for('login', next=request.url)) @@ -1398,6 +1397,7 @@ def indieauth_endpoint(): abort(403) return + session['logged_in'] = True me = auth['me'] state = auth['state'] scope = ' '.join(auth['scope']) From d4bf73756f4d1939051d7f340bc71c140c2e4285 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 7 Jun 2018 00:00:35 +0200 Subject: [PATCH 0105/1425] [WIP] Start a new ActivityPub module --- little_boxes/__init__.py | 0 little_boxes/activitypub.py | 1042 +++++++++++++++++++++++++++++++++ little_boxes/errors.py | 49 ++ little_boxes/remote_object.py | 39 ++ little_boxes/urlutils.py | 47 ++ little_boxes/utils.py | 62 ++ 6 files changed, 1239 insertions(+) create mode 100644 little_boxes/__init__.py create mode 100644 little_boxes/activitypub.py create mode 100644 little_boxes/errors.py create mode 100644 little_boxes/remote_object.py create mode 100644 little_boxes/urlutils.py create mode 100644 little_boxes/utils.py diff --git a/little_boxes/__init__.py b/little_boxes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/little_boxes/activitypub.py b/little_boxes/activitypub.py new file mode 100644 index 0000000..8205a26 --- /dev/null +++ b/little_boxes/activitypub.py @@ -0,0 +1,1042 @@ +"""Core ActivityPub classes.""" +import logging +import json +import binascii +import os +from datetime import datetime +from enum import Enum + +from .errors import BadActivityError +from .errors import UnexpectedActivityTypeError +from .errors import NotFromOutboxError +from . import utils +from .remote_object import OBJECT_FETCHER + +from typing import List +from typing import Optional +from typing import Dict +from typing import Any +from typing import Union +from typing import Type + +logger = logging.getLogger(__name__) + +# Helper/shortcut for typing +ObjectType = Dict[str, Any] +ObjectOrIDType = Union[str, ObjectType] + +CTX_AS = 'https://www.w3.org/ns/activitystreams' +CTX_SECURITY = 'https://w3id.org/security/v1' +AS_PUBLIC = 'https://www.w3.org/ns/activitystreams#Public' + +COLLECTION_CTX = [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "Hashtag": "as:Hashtag", + "sensitive": "as:sensitive", + } +] + +# Will be used to keep track of all the defined activities +_ACTIVITY_CLS: Dict['ActivityTypeEnum', Type['_BaseActivity']] = {} + + +class ActivityType(Enum): + """Supported activity `type`.""" + ANNOUNCE = 'Announce' + BLOCK = 'Block' + LIKE = 'Like' + CREATE = 'Create' + UPDATE = 'Update' + PERSON = 'Person' + ORDERED_COLLECTION = 'OrderedCollection' + ORDERED_COLLECTION_PAGE = 'OrderedCollectionPage' + COLLECTION_PAGE = 'CollectionPage' + COLLECTION = 'Collection' + NOTE = 'Note' + ACCEPT = 'Accept' + REJECT = 'Reject' + FOLLOW = 'Follow' + DELETE = 'Delete' + UNDO = 'Undo' + IMAGE = 'Image' + TOMBSTONE = 'Tombstone' + + +def parse_activity(payload: ObjectType, expected: Optional[ActivityType] = None) -> 'BaseActivity': + t = ActivityType(payload['type']) + + if expected and t != expected: + raise UnexpectedActivityTypeError(f'expected a {expected.name} activity, got a {payload["type"]}') + + if t not in _ACTIVITY_CLS: + raise BadActivityError(f'unsupported activity type {payload["type"]}') + + activity = _ACTIVITY_CLS[t](**payload) + + return activity + + +def random_object_id() -> str: + """Generates a random object ID.""" + return binascii.hexlify(os.urandom(8)).decode('utf-8') + + +def _to_list(data: Union[List[Any], Any]) -> List[Any]: + """Helper to convert fields that can be either an object or a list of objects to a list of object.""" + if isinstance(data, list): + return data + return [data] + + +def clean_activity(activity: ObjectType) -> Dict[str, Any]: + """Clean the activity before rendering it. + - Remove the hidden bco and bcc field + """ + for field in ['bto', 'bcc']: + if field in activity: + del(activity[field]) + if activity['type'] == 'Create' and field in activity['object']: + del(activity['object'][field]) + return activity + + +def _get_actor_id(actor: ObjectOrIDType) -> str: + """Helper for retrieving an actor `id`.""" + if isinstance(actor, dict): + return actor['id'] + return actor + + +class _ActivityMeta(type): + """Metaclass for keeping track of subclass.""" + def __new__(meta, name, bases, class_dict): + cls = type.__new__(meta, name, bases, class_dict) + + # Ensure the class has an activity type defined + if not cls.ACTIVITY_TYPE: + raise ValueError(f'class {name} has no ACTIVITY_TYPE') + + # Register it + _REGISTER[cls.ACTIVITY_TYPE] = cls + return cls + + +class _BaseActivity(object, metaclass=_ActivityMeta): + """Base class for ActivityPub activities.""" + + ACTIVITY_TYPE: Optional[ActivityType] = None # the ActivityTypeEnum the class will represent + OBJECT_REQUIRED = False # Whether the object field is required or note + ALLOWED_OBJECT_TYPES: List[ActivityType] = [] # + ACTOR_REQUIRED = True # Most of the object requires an actor, so this flag in on by default + + def __init__(self, **kwargs) -> None: + if kwargs.get('type') and kwargs.pop('type') != self.ACTIVITY_TYPE.value: + raise UnexpectedActivityTypeError(f'Expect the type to be {self.ACTIVITY_TYPE.value!r}') + + # Initialize the dict that will contains all the activity fields + self._data: Dict[str, Any] = { + 'type': self.ACTIVITY_TYPE.value + } + logger.debug(f'initializing a {self.ACTIVITY_TYPE.value} activity: {kwargs}') + + # The id may not be present for new activities + if 'id' in kwargs: + self._data['id'] = kwargs.pop('id') + + if self.ACTIVITY_TYPE != ActivityType.PERSON and self.ACTOR_REQUIRED: + actor = kwargs.get('actor') + if actor: + kwargs.pop('actor') + actor = self._validate_person(actor) + self._data['actor'] = actor + else: + raise BadActivityError('missing actor') + + if self.OBJECT_REQUIRED and 'object' in kwargs: + obj = kwargs.pop('object') + if isinstance(obj, str): + # The object is a just a reference the its ID/IRI + # FIXME(tsileo): fetch the ref + self._data['object'] = obj + else: + if not self.ALLOWED_OBJECT_TYPES: + raise UnexpectedActivityTypeError('unexpected object') + if 'type' not in obj or (self.ACTIVITY_TYPE != ActivityType.CREATE and 'id' not in obj): + raise BadActivityError('invalid object, missing type') + if ActivityType(obj['type']) not in self.ALLOWED_OBJECT_TYPES: + raise UnexpectedActivityTypeError( + f'unexpected object type {obj["type"]} (allowed={self.ALLOWED_OBJECT_TYPES!r})' + ) + self._data['object'] = obj + + if '@context' not in kwargs: + self._data['@context'] = CTX_AS + else: + self._data['@context'] = kwargs.pop('@context') + + # @context check + if not isinstance(self._data['@context'], list): + self._data['@context'] = [self._data['@context']] + if CTX_SECURITY not in self._data['@context']: + self._data['@context'].append(CTX_SECURITY) + if isinstance(self._data['@context'][-1], dict): + self._data['@context'][-1]['Hashtag'] = 'as:Hashtag' + self._data['@context'][-1]['sensitive'] = 'as:sensitive' + else: + self._data['@context'].append({'Hashtag': 'as:Hashtag', 'sensitive': 'as:sensitive'}) + + # FIXME(tsileo): keys required for all subclasses? + allowed_keys = None + try: + allowed_keys = self._init(**kwargs) + logger.debug('calling custom init') + except NotImplementedError: + pass + + if allowed_keys: + # Allows an extra to (like for Accept and Follow) + kwargs.pop('to', None) + if len(set(kwargs.keys()) - set(allowed_keys)) > 0: + raise BadActivityError(f'extra data left: {kwargs!r}') + else: + # Remove keys with `None` value + valid_kwargs = {} + for k, v in kwargs.items(): + if v is None: + continue + valid_kwargs[k] = v + self._data.update(**valid_kwargs) + + def _init(self, **kwargs) -> Optional[List[str]]: + """Optional init callback that may returns a list of allowed keys.""" + raise NotImplementedError + + def __repr__(self) -> str: + """Pretty repr.""" + return '{}({!r})'.format(self.__class__.__qualname__, self._data.get('id')) + + def __str__(self) -> str: + """Returns the ID/IRI when castign to str.""" + return str(self._data.get('id', f'[new {self.ACTIVITY_TYPE} activity]')) + + def __getattr__(self, name: str) -> Any: + """Allow to access the object field as regular attributes.""" + if self._data.get(name): + return self._data.get(name) + + def _outbox_set_id(self, uri: str, obj_id: str) -> None: + """Optional callback for subclasses to so something with a newly generated ID (for outbox activities).""" + raise NotImplementedError + + def outbox_set_id(self, uri: str, obj_id: str) -> None: + """Set the ID for a new activity.""" + logger.debug(f'setting ID {uri} / {obj_id}') + self._data['id'] = uri + try: + self._set_id(uri, obj_id) + except NotImplementedError: + pass + + def _actor_id(self, obj: ObjectOrIDType) -> str: + if isinstance(obj, dict) and obj['type'] == ActivityType.PERSON.value: + obj_id = obj.get('id') + if not obj_id: + raise BadActivityError(f'missing object id: {obj!r}') + return obj_id + elif isinstance(obj, str): + return obj + else: + raise BadActivityError(f'invalid "actor" field: {obj!r}') + + def _validate_person(self, obj: ObjectOrIDType) -> str: + obj_id = self._actor_id(obj) + try: + actor = OBJECT_FETCHER.fetch(obj_id) + except Exception: + raise BadActivityError(f'failed to validate actor {obj!r}') + + if not actor or not 'id' in actor: + raise BadActivityError(f'invalid actor {actor}') + + return actor['id'] + + def get_object(self) -> 'BaseActivity': + """Returns the object as a BaseActivity instance.""" + if self.__obj: + return self.__obj + if isinstance(self._data['object'], dict): + p = parse_activity(self._data['object']) + else: + obj = OBJECT_FETCHER.fetch(self._data['object']) + if ActivityType(obj.get('type')) not in self.ALLOWED_OBJECT_TYPES: + raise UnexpectedActivityTypeError(f'invalid object type {obj.get("type")!r}') + p = parse_activity(obj) + + self.__obj: Optional[BaseActivity] = p + return p + + def reset_object_cache(self) -> None: + self.__obj = None + + def to_dict(self, embed: bool = False, embed_object_id_only: bool = False) -> ObjectType: + """Serializes the activity back to a dict, ready to be JSON serialized.""" + data = dict(self._data) + if embed: + for k in ['@context', 'signature']: + if k in data: + del(data[k]) + if data.get('object') and embed_object_id_only and isinstance(data['object'], dict): + try: + data['object'] = data['object']['id'] + except KeyError: + raise BadActivityError(f'embedded object {data["object"]!r} should have an id') + + return data + + def get_actor(self) -> 'BaseActivity': + # FIXME(tsileo): cache the actor (same way as get_object) + actor = self._data.get('actor') + if not actor and self.ACTOR_REQUIRED: + # Quick hack for Note objects + if self.ACTIVITY_TYPE == ActivityType.NOTE: + actor = str(self._data.get('attributedTo')) + else: + raise BadActivityError(f'failed to fetch actor: {self._data!r}') + + actor_id = self._actor_id(actor) + return Person(**OBJECT_FETCHER.fetch(actor_id)) + + def _pre_post_to_outbox(self) -> None: + raise NotImplementedError + + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: + raise NotImplementedError + + def _undo_outbox(self) -> None: + raise NotImplementedError + + def _pre_process_from_inbox(self) -> None: + raise NotImplementedError + + def _process_from_inbox(self) -> None: + raise NotImplementedError + + def _undo_inbox(self) -> None: + raise NotImplementedError + + # FIXME(tsileo): delete these? + def _undo_should_purge_cache(self) -> bool: + raise NotImplementedError + + def _should_purge_cache(self) -> bool: + raise NotImplementedError + + def process_from_inbox(self) -> None: + logger.debug(f'calling main process from inbox hook for {self}') + actor = self.get_actor() + + # Check for Block activity + # ABC + if self.outbox_is_blocked(actor.id): + # TODO(tsileo): raise ActorBlockedError + logger.info(f'actor {actor!r} is blocked, dropping the received activity {self!r}') + return + + # ABC + if self.inbox_get_by_id(self.id): + # The activity is already in the inbox + logger.info(f'received duplicate activity {self}, dropping it') + return + + try: + self._pre_process_from_inbox() + logger.debug('called pre process from inbox hook') + except NotImplementedError: + logger.debug('pre process from inbox hook not implemented') + + # ABC + self.inbox_create(self) + logger.info('activity {self!r} saved') + + try: + self._process_from_inbox() + logger.debug('called process from inbox hook') + except NotImplementedError: + logger.debug('process from inbox hook not implemented') + + def post_to_outbox(self) -> None: + logger.debug(f'calling main post to outbox hook for {self}') + + # Assign create a random ID + obj_id = random_object_id() + self.set_id(f'{ID}/outbox/{obj_id}', obj_id) + + try: + self._pre_post_to_outbox() + logger.debug(f'called pre post to outbox hook') + except NotImplementedError: + logger.debug('pre post to outbox hook not implemented') + + # ABC + self.outbox_create(self) + + recipients = self.recipients() + logger.info(f'recipients={recipients}') + activity = clean_activity(activity) + + try: + self._post_to_outbox(obj_id, activity, recipients) + logger.debug(f'called post to outbox hook') + except NotImplementedError: + logger.debug('post to outbox hook not implemented') + + payload = json.dumps(activity) + for recp in recipients: + logger.debug(f'posting to {recp}') + + # ABC + self.post_to_remote_inbox(payload, recp) + + def _post_to_inbox(self, payload: str, to: str): + tasks.post_to_inbox.delay(payload, to) + + def _recipients(self) -> List[str]: + return [] + + def recipients(self) -> List[str]: + recipients = self._recipients() + actor_id = self.get_actor().id + + out: List[str] = [] + for recipient in recipients: + # if recipient in PUBLIC_INSTANCES: + # if recipient not in out: + # out.append(str(recipient)) + # continue + if recipient in [actor_id, AS_PUBLIC, None]: + continue + if isinstance(recipient, Person): + if recipient.id == actor_id: + continue + actor = recipient + else: + raw_actor = OBJECT_FETCHER.fetch(recipient) + if raw_actor['type'] == ActiivtyType.PERSON.name: + actor = Person(**raw_actor) + + if actor.endpoints: + shared_inbox = actor.endpoints.get('sharedInbox') + if shared_inbox not in out: + out.append(shared_inbox) + continue + + if actor.inbox and actor.inbox not in out: + out.append(actor.inbox) + + # Is the activity a `Collection`/`OrderedCollection`? + elif raw_actor['type'] in [ActivityType.COLLECTION.value, + ActivityType.ORDERED_COLLECTION.value]: + for item in parse_collection(raw_actor): + if item in [ME, AS_PUBLIC]: + continue + try: + col_actor = Person(**OBJECT_FETCHER.fetch(item)) + except NotAnActorError: + pass + + if col_actor.endpoints: + shared_inbox = col_actor.endpoints.get('sharedInbox') + if shared_inbox not in out: + out.append(shared_inbox) + continue + if col_actor.inbox and col_actor.inbox not in out: + out.append(col_actor.inbox) + else: + raise BadActivityError(f'failed to parse {raw_actor!r}') + + return out + + def build_undo(self) -> 'BaseActivity': + raise NotImplementedError + + def build_delete(self) -> 'BaseActivity': + raise NotImplementedError + + +class Person(BaseActivity): + ACTIVITY_TYPE = ActivityType.PERSON + OBJECT_REQUIRED = False + ACTOR_REQUIRED = False + + +class Block(BaseActivity): + ACTIVITY_TYPE = ActivityType.BLOCK + OBJECT_REQUIRED = True + ACTOR_REQUIRED = True + + +class Collection(BaseActivity): + ACTIVITY_TYPE = ActivityType.COLLECTION + OBJECT_REQUIRED = False + ACTOR_REQUIRED = False + + +class Image(BaseActivity): + ACTIVITY_TYPE = ActivityType.IMAGE + OBJECT_REQUIRED = False + ACTOR_REQUIRED = False + + def _init(self, **kwargs): + self._data.update( + url=kwargs.pop('url'), + ) + + def __repr__(self): + return 'Image({!r})'.format(self._data.get('url')) + + +class Follow(BaseActivity): + ACTIVITY_TYPE = ActivityType.FOLLOW + ALLOWED_OBJECT_TYPES = [ActivityType.PERSON] + OBJECT_REQUIRED = True + ACTOR_REQUIRED = True + + + def _build_reply(self, reply_type: ActivityType) -> BaseActivity: + if reply_type == ActivityType.ACCEPT: + return Accept( + object=self.to_dict(embed=True), + ) + + raise ValueError(f'type {reply_type} is invalid for building a reply') + + def _recipients(self) -> List[str]: + return [self.get_object().id] + + def _process_from_inbox(self) -> None: + """Receiving a Follow should trigger an Accept.""" + accept = self.build_accept() + accept.post_to_outbox() + + remote_actor = self.get_actor().id + + # ABC + self.new_follower(remote_actor) + + def _undo_inbox(self) -> None: + # ABC + self.undo_new_follower(self.get_actor().id) + + def _undo_outbox(self) -> None: + # ABC + self.undo_new_following(self.get_object().id) + + def build_accept(self) -> BaseActivity: + return self._build_reply(ActivityType.ACCEPT) + + def build_undo(self) -> BaseActivity: + return Undo(object=self.to_dict(embed=True)) + + +class Accept(BaseActivity): + ACTIVITY_TYPE = ActivityType.ACCEPT + ALLOWED_OBJECT_TYPES = [ActivityType.FOLLOW] + OBJECT_REQUIRED = True + ACTOR_REQUIRED = True + + def _recipients(self) -> List[str]: + return [self.get_object().get_actor().id] + + def _pre_process_from_inbox(self) -> None: + # FIXME(tsileo): ensure the actor match the object actor + + def _process_from_inbox(self) -> None: + # ABC + self.new_following(self.get_actor().id) + + +class Undo(BaseActivity): + ACTIVITY_TYPE = ActivityType.UNDO + ALLOWED_OBJECT_TYPES = [ActivityType.FOLLOW, ActivityType.LIKE, ActivityType.ANNOUNCE] + OBJECT_REQUIRED = True + ACTOR_REQUIRED = True + + def _recipients(self) -> List[str]: + obj = self.get_object() + if obj.type_enum == ActivityType.FOLLOW: + return [obj.get_object().id] + else: + return [obj.get_object().get_actor().id] + # TODO(tsileo): handle like and announce + raise Exception('TODO') + + def _pre_process_from_inbox(self) -> None: + """Ensures an Undo activity comes from the same actor as the updated activity.""" + obj = self.get_object() + actor = self.get_actor() + if actor.id != obj.get_actor().id: + raise BadActivityError(f'{actor!r} cannot update {obj!r}') + + def _process_from_inbox(self) -> None: + obj = self.get_object() + # FIXME(tsileo): move this to _undo_inbox impl + # DB.inbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) + + try: + obj._undo_inbox() + except NotImplementedError: + pass + + def _pre_post_to_outbox(self) -> None: + """Ensures an Undo activity references an activity owned by the instance.""" + obj = self.get_object() + if not obj.id.startswith(ID): + raise NotFromOutboxError(f'object {obj["id"]} is not owned by this instance') + + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: + logger.debug('processing undo to outbox') + logger.debug('self={}'.format(self)) + obj = self.get_object() + logger.debug('obj={}'.format(obj)) + + # FIXME(tsileo): move this to _undo_inbox impl + # DB.outbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) + + try: + obj._undo_outbox() + logger.debug(f'_undo_outbox called for {obj}') + except NotImplementedError: + logger.debug(f'_undo_outbox not implemented for {obj}') + pass + + +class Like(BaseActivity): + ACTIVITY_TYPE = ActivityType.LIKE + ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] + OBJECT_REQUIRED = True + ACTOR_REQUIRED = True + + def _recipients(self) -> List[str]: + return [self.get_object().get_actor().id] + + def _process_from_inbox(self): + obj = self.get_object() + # Update the meta counter if the object is published by the server + # FIXME(tsileo): continue here + DB.outbox.update_one({'activity.object.id': obj.id}, { + '$inc': {'meta.count_like': 1}, + }) + # XXX(tsileo): notification?? + + def _undo_inbox(self) -> None: + obj = self.get_object() + # Update the meta counter if the object is published by the server + DB.outbox.update_one({'activity.object.id': obj.id}, { + '$inc': {'meta.count_like': -1}, + }) + + def _undo_should_purge_cache(self) -> bool: + # If a like coutn was decremented, we need to purge the application cache + return self.get_object().id.startswith(BASE_URL) + + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]): + obj = self.get_object() + # Unlikely, but an actor can like it's own post + DB.outbox.update_one({'activity.object.id': obj.id}, { + '$inc': {'meta.count_like': 1}, + }) + + # Keep track of the like we just performed + DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.liked': obj_id}}) + + def _undo_outbox(self) -> None: + obj = self.get_object() + # Unlikely, but an actor can like it's own post + DB.outbox.update_one({'activity.object.id': obj.id}, { + '$inc': {'meta.count_like': -1}, + }) + + DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.liked': False}}) + + def build_undo(self) -> BaseActivity: + return Undo(object=self.to_dict(embed=True, embed_object_id_only=True)) + + +class Announce(BaseActivity): + ACTIVITY_TYPE = ActivityType.ANNOUNCE + ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] + + def _recipients(self) -> List[str]: + recipients = [] + + for field in ['to', 'cc']: + if field in self._data: + recipients.extend(_to_list(self._data[field])) + + return recipients + + def _process_from_inbox(self) -> None: + if isinstance(self._data['object'], str) and not self._data['object'].startswith('http'): + # TODO(tsileo): actually drop it without storing it and better logging, also move the check somewhere else + logger.warn( + f'received an Annouce referencing an OStatus notice ({self._data["object"]}), dropping the message' + ) + return + # Save/cache the object, and make it part of the stream so we can fetch it + if isinstance(self._data['object'], str): + raw_obj = OBJECT_SERVICE.get( + self._data['object'], + reload_cache=True, + part_of_stream=True, + announce_published=self._data['published'], + ) + obj = parse_activity(raw_obj) + else: + obj = self.get_object() + + DB.outbox.update_one({'activity.object.id': obj.id}, { + '$inc': {'meta.count_boost': 1}, + }) + + def _undo_inbox(self) -> None: + obj = self.get_object() + # Update the meta counter if the object is published by the server + DB.outbox.update_one({'activity.object.id': obj.id}, { + '$inc': {'meta.count_boost': -1}, + }) + + def _undo_should_purge_cache(self) -> bool: + # If a like coutn was decremented, we need to purge the application cache + return self.get_object().id.startswith(BASE_URL) + + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: + if isinstance(self._data['object'], str): + # Put the object in the cache + OBJECT_SERVICE.get( + self._data['object'], + reload_cache=True, + part_of_stream=True, + announce_published=self._data['published'], + ) + + obj = self.get_object() + DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.boosted': obj_id}}) + + def _undo_outbox(self) -> None: + obj = self.get_object() + DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.boosted': False}}) + + def build_undo(self) -> BaseActivity: + return Undo(object=self.to_dict(embed=True)) + + +class Delete(BaseActivity): + ACTIVITY_TYPE = ActivityType.DELETE + ALLOWED_OBJECT_TYPES = [ActivityType.NOTE, ActivityType.TOMBSTONE] + OBJECT_REQUIRED = True + + def _get_actual_object(self) -> BaseActivity: + obj = self.get_object() + if obj.type_enum == ActivityType.TOMBSTONE: + obj = parse_activity(OBJECT_SERVICE.get(obj.id)) + return obj + + def _recipients(self) -> List[str]: + obj = self._get_actual_object() + return obj._recipients() + + def _pre_process_from_inbox(self) -> None: + """Ensures a Delete activity comes from the same actor as the deleted activity.""" + obj = self._get_actual_object() + actor = self.get_actor() + if actor.id != obj.get_actor().id: + raise BadActivityError(f'{actor!r} cannot delete {obj!r}') + + def _process_from_inbox(self) -> None: + DB.inbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) + obj = self._get_actual_object() + if obj.type_enum == ActivityType.NOTE: + obj._delete_from_threads() + + # TODO(tsileo): also purge the cache if it's a reply of a published activity + + def _pre_post_to_outbox(self) -> None: + """Ensures the Delete activity references a activity from the outbox (i.e. owned by the instance).""" + obj = self._get_actual_object() + if not obj.id.startswith(ID): + raise NotFromOutboxError(f'object {obj["id"]} is not owned by this instance') + + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: + DB.outbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) + + +class Update(BaseActivity): + ACTIVITY_TYPE = ActivityType.UPDATE + ALLOWED_OBJECT_TYPES = [ActivityType.NOTE, ActivityType.PERSON] + OBJECT_REQUIRED = True + + def _pre_process_from_inbox(self) -> None: + """Ensures an Update activity comes from the same actor as the updated activity.""" + obj = self.get_object() + actor = self.get_actor() + if actor.id != obj.get_actor().id: + raise BadActivityError(f'{actor!r} cannot update {obj!r}') + + def _process_from_inbox(self): + obj = self.get_object() + if obj.type_enum == ActivityType.NOTE: + DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'activity.object': obj.to_dict()}}) + return + + # If the object is a Person, it means the profile was updated, we just refresh our local cache + ACTOR_SERVICE.get(obj.id, reload_cache=True) + + # TODO(tsileo): implements _should_purge_cache if it's a reply of a published activity (i.e. in the outbox) + + def _pre_post_to_outbox(self) -> None: + obj = self.get_object() + if not obj.id.startswith(ID): + raise NotFromOutboxError(f'object {obj["id"]} is not owned by this instance') + + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: + obj = self._data['object'] + + update_prefix = 'activity.object.' + update: Dict[str, Any] = {'$set': dict(), '$unset': dict()} + update['$set'][f'{update_prefix}updated'] = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' + for k, v in obj.items(): + if k in ['id', 'type']: + continue + if v is None: + update['$unset'][f'{update_prefix}{k}'] = '' + else: + update['$set'][f'{update_prefix}{k}'] = v + + if len(update['$unset']) == 0: + del(update['$unset']) + + print(f'updating note from outbox {obj!r} {update}') + logger.info(f'updating note from outbox {obj!r} {update}') + DB.outbox.update_one({'activity.object.id': obj['id']}, update) + # FIXME(tsileo): should send an Update (but not a partial one, to all the note's recipients + # (create a new Update with the result of the update, and send it without saving it?) + + +class Create(BaseActivity): + ACTIVITY_TYPE = ActivityType.CREATE + ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] + OBJECT_REQUIRED = True + + def _set_id(self, uri: str, obj_id: str) -> None: + self._data['object']['id'] = uri + '/activity' + self._data['object']['url'] = ID + '/' + self.get_object().type.lower() + '/' + obj_id + self.reset_object_cache() + + def _init(self, **kwargs): + obj = self.get_object() + if not obj.attributedTo: + self._data['object']['attributedTo'] = self.get_actor().id + if not obj.published: + if self.published: + self._data['object']['published'] = self.published + else: + now = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' + self._data['published'] = now + self._data['object']['published'] = now + + def _recipients(self) -> List[str]: + # TODO(tsileo): audience support? + recipients = [] + for field in ['to', 'cc', 'bto', 'bcc']: + if field in self._data: + recipients.extend(_to_list(self._data[field])) + + recipients.extend(self.get_object()._recipients()) + + return recipients + + def _update_threads(self) -> None: + logger.debug('_update_threads hook') + obj = self.get_object() + + # TODO(tsileo): re-enable me + # tasks.fetch_og.delay('INBOX', self.id) + + threads = [] + reply = obj.get_local_reply() + print(f'initial_reply={reply}') + print(f'{obj}') + logger.debug(f'initial_reply={reply}') + reply_id = None + direct_reply = 1 + while reply is not None: + if not DB.inbox.find_one_and_update({'activity.object.id': reply.id}, { + '$inc': { + 'meta.count_reply': 1, + 'meta.count_direct_reply': direct_reply, + }, + '$addToSet': {'meta.thread_children': obj.id}, + }): + DB.outbox.update_one({'activity.object.id': reply.id}, { + '$inc': { + 'meta.count_reply': 1, + 'meta.count_direct_reply': direct_reply, + }, + '$addToSet': {'meta.thread_children': obj.id}, + }) + + direct_reply = 0 + reply_id = reply.id + reply = reply.get_local_reply() + logger.debug(f'next_reply={reply}') + threads.append(reply_id) + # FIXME(tsileo): obj.id is None!! + print(f'reply_id={reply_id} {obj.id} {obj._data} {self.id}') + + if reply_id: + if not DB.inbox.find_one_and_update({'activity.object.id': obj.id}, { + '$set': { + 'meta.thread_parents': threads, + 'meta.thread_root_parent': reply_id, + }, + }): + DB.outbox.update_one({'activity.object.id': obj.id}, { + '$set': { + 'meta.thread_parents': threads, + 'meta.thread_root_parent': reply_id, + }, + }) + logger.debug('_update_threads done') + + def _process_from_inbox(self) -> None: + self._update_threads() + + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: + self._update_threads() + + def _should_purge_cache(self) -> bool: + # TODO(tsileo): handle reply of a reply... + obj = self.get_object() + in_reply_to = obj.inReplyTo + if in_reply_to: + local_activity = DB.outbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) + if local_activity: + return True + + return False + + +class Tombstone(BaseActivity): + ACTIVITY_TYPE = ActivityType.TOMBSTONE + + +class Note(BaseActivity): + ACTIVITY_TYPE = ActivityType.NOTE + + def _init(self, **kwargs): + print(self._data) + # Remove the `actor` field as `attributedTo` is used for `Note` instead + if 'actor' in self._data: + del(self._data['actor']) + if 'sensitive' not in kwargs: + self._data['sensitive'] = False + + def _recipients(self) -> List[str]: + # TODO(tsileo): audience support? + recipients: List[str] = [] + + # If the note is public, we publish it to the defined "public instances" + if AS_PUBLIC in self._data.get('to', []): + recipients.extend(PUBLIC_INSTANCES) + print('publishing to public instances') + print(recipients) + + for field in ['to', 'cc', 'bto', 'bcc']: + if field in self._data: + recipients.extend(_to_list(self._data[field])) + + return recipients + + def _delete_from_threads(self) -> None: + logger.debug('_delete_from_threads hook') + + reply = self.get_local_reply() + logger.debug(f'initial_reply={reply}') + direct_reply = -1 + while reply is not None: + if not DB.inbox.find_one_and_update({'activity.object.id': reply.id}, { + '$inc': { + 'meta.count_reply': -1, + 'meta.count_direct_reply': direct_reply, + }, + '$pull': {'meta.thread_children': self.id}, + + }): + DB.outbox.update_one({'activity.object.id': reply.id}, { + '$inc': { + 'meta.count_reply': 1, + 'meta.count_direct_reply': direct_reply, + }, + '$pull': {'meta.thread_children': self.id}, + }) + + direct_reply = 0 + reply = reply.get_local_reply() + logger.debug(f'next_reply={reply}') + + logger.debug('_delete_from_threads done') + return None + + def get_local_reply(self) -> Optional[BaseActivity]: + "Find the note reply if any.""" + in_reply_to = self.inReplyTo + if not in_reply_to: + # This is the root comment + return None + + inbox_parent = DB.inbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) + if inbox_parent: + return parse_activity(inbox_parent['activity']['object']) + + outbox_parent = DB.outbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) + if outbox_parent: + return parse_activity(outbox_parent['activity']['object']) + + # The parent is no stored on this instance + return None + + def build_create(self) -> BaseActivity: + """Wraps an activity in a Create activity.""" + create_payload = { + 'object': self.to_dict(embed=True), + 'actor': self.attributedTo or ME, + } + for field in ['published', 'to', 'bto', 'cc', 'bcc', 'audience']: + if field in self._data: + create_payload[field] = self._data[field] + + return Create(**create_payload) + + def build_like(self) -> BaseActivity: + return Like(object=self.id) + + def build_announce(self) -> BaseActivity: + return Announce( + object=self.id, + to=[AS_PUBLIC], + cc=[ID+'/followers', self.attributedTo], + published=datetime.utcnow().replace(microsecond=0).isoformat() + 'Z', + ) + + def build_delete(self) -> BaseActivity: + return Delete(object=Tombstone(id=self.id).to_dict(embed=True)) + + def get_tombstone(self, deleted: Optional[str]) -> BaseActivity: + return Tombstone( + id=self.id, + published=self.published, + deleted=deleted, + updated=deleted, + ) diff --git a/little_boxes/errors.py b/little_boxes/errors.py new file mode 100644 index 0000000..f70159a --- /dev/null +++ b/little_boxes/errors.py @@ -0,0 +1,49 @@ +"""Errors raised by this package.""" +from typing import Optional +from typing import Dict +from typing import Any + + +class Error(Exception): + """HTTP-friendly base error, with a status code, a message and an optional payload.""" + status_code = 400 + + def __init__(self, message: str, status_code: Optional[int] = None, payload: Optional[Dict[str, Any]] = None) -> None: + Exception.__init__(self) + self.message = message + if status_code is not None: + self.status_code = status_code + self.payload = payload + + def to_dict(self) -> Dict[str, Any]: + rv = dict(self.payload or ()) + rv['message'] = self.message + return rv + + def __repr__(self) -> str: + return f'{self.__class__.__qualname__}({self.message!r}, payload={self.payload!r}, status_code={self.status_code})' + + +class ActorBlockedError(Error): + """Raised when an activity from a blocked actor is received.""" + + +class NotFromOutboxError(Error): + """Raised when an activity targets an object from the inbox when an object from the oubox was expected.""" + + +class ActivityNotFoundError(Error): + """Raised when an activity is not found.""" + status_code = 404 + + +class BadActivityError(Error): + """Raised when an activity could not be parsed/initialized.""" + + +class RecursionLimitExceededError(BadActivityError): + """Raised when the recursion limit for fetching remote object was exceeded (likely a collection).""" + + +class UnexpectedActivityTypeError(BadActivityError): + """Raised when an another activty was expected.""" diff --git a/little_boxes/remote_object.py b/little_boxes/remote_object.py new file mode 100644 index 0000000..9a528bf --- /dev/null +++ b/little_boxes/remote_object.py @@ -0,0 +1,39 @@ +import logging +from typing import Any + +import requests +from urllib.parse import urlparse +from Crypto.PublicKey import RSA + +from .urlutils import check_url +from .errors import ActivityNotFoundError +from .errors import UnexpectedActivityTypeError + +logger = logging.getLogger(__name__) + + +class DefaultRemoteObjectFetcher(object): + """Not meant to be used on production, a caching layer, and DB shortcut fox inbox/outbox should be hooked.""" + + def __init__(self): + self._user_agent = 'Little Boxes (+https://github.com/tsileo/little_boxes)' + + def fetch(self, iri): + check_url(iri) + + resp = requests.get(actor_url, headers={ + 'Accept': 'application/activity+json', + 'User-Agent': self._user_agent, + }) + + if resp.status_code == 404: + raise ActivityNotFoundError(f'{actor_url} cannot be fetched, 404 not found error') + + resp.raise_for_status() + + return resp.json() + +OBJECT_FETCHER = DefaultRemoteObjectFetcher() + +def set_object_fetcher(object_fetcher: Any): + OBJECT_FETCHER = object_fetcher diff --git a/little_boxes/urlutils.py b/little_boxes/urlutils.py new file mode 100644 index 0000000..99f900d --- /dev/null +++ b/little_boxes/urlutils.py @@ -0,0 +1,47 @@ +import logging +import os +import socket +import ipaddress +from urllib.parse import urlparse + +from . import strtobool +from .errors import Error + +logger = logging.getLogger(__name__) + + +class InvalidURLError(Error): + pass + + +def is_url_valid(url: str) -> bool: + parsed = urlparse(url) + if parsed.scheme not in ['http', 'https']: + return False + + # XXX in debug mode, we want to allow requests to localhost to test the federation with local instances + debug_mode = strtobool(os.getenv('MICROBLOGPUB_DEBUG', 'false')) + if debug_mode: + return True + + if parsed.hostname in ['localhost']: + return False + + try: + ip_address = socket.getaddrinfo(parsed.hostname, parsed.port or 80)[0][4][0] + except socket.gaierror: + logger.exception(f'failed to lookup url {url}') + return False + + if ipaddress.ip_address(ip_address).is_private: + logger.info(f'rejecting private URL {url}') + return False + + return True + + +def check_url(url: str) -> None: + if not is_url_valid(url): + raise InvalidURLError(f'"{url}" is invalid') + + return None diff --git a/little_boxes/utils.py b/little_boxes/utils.py new file mode 100644 index 0000000..64987c2 --- /dev/null +++ b/little_boxes/utils.py @@ -0,0 +1,62 @@ +"""Contains some ActivityPub related utils.""" +from typing import Optional +from typing import Dict +from typing import List +from typing import Any + +import requests + +from .errors import RecursionLimitExceededError +from .errors import UnexpectedActivityTypeError +from .remote_object import OBJECT_FETCHER + + +def parse_collection( + payload: Optional[Dict[str, Any]] = None, + url: Optional[str] = None, + level: int = 0, +) -> List[Any]: + """Resolve/fetch a `Collection`/`OrderedCollection`.""" + if level > 3: + raise RecursionLimitExceededError('recursion limit exceeded') + + # Go through all the pages + headers = {'Accept': 'application/activity+json'} + if user_agent: + headers['User-Agent'] = user_agent + + out: List[Any] = [] + if url: + payload = OBJECT_FETCHER.fetch(url) + if not payload: + raise ValueError('must at least prove a payload or an URL') + + if payload['type'] in ['Collection', 'OrderedCollection']: + if 'orderedItems' in payload: + return payload['orderedItems'] + if 'items' in payload: + return payload['items'] + if 'first' in payload: + if 'orderedItems' in payload['first']: + out.extend(payload['first']['orderedItems']) + if 'items' in payload['first']: + out.extend(payload['first']['items']) + n = payload['first'].get('next') + if n: + out.extend(parse_collection(url=n, level=level+1)) + return out + + while payload: + if payload['type'] in ['CollectionPage', 'OrderedCollectionPage']: + if 'orderedItems' in payload: + out.extend(payload['orderedItems']) + if 'items' in payload: + out.extend(payload['items']) + n = payload.get('next') + if n is None: + break + payload = OBJECT_FETCHER.fetch(n) + else: + raise UnexpectedActivityTypeError('unexpected activity type {}'.format(payload['type'])) + + return out From 2abf0fffd20394347f0a7cc54f315115d4c16879 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 8 Jun 2018 21:33:46 +0200 Subject: [PATCH 0106/1425] [WIP] More work on the ActivityPub module --- little_boxes/__init__.py | 12 + little_boxes/activitypub.py | 428 +++++++++++++--------------------- little_boxes/errors.py | 10 +- little_boxes/remote_object.py | 14 +- little_boxes/utils.py | 5 - 5 files changed, 187 insertions(+), 282 deletions(-) diff --git a/little_boxes/__init__.py b/little_boxes/__init__.py index e69de29..c30c37d 100644 --- a/little_boxes/__init__.py +++ b/little_boxes/__init__.py @@ -0,0 +1,12 @@ +import logging + +logger = logging.getLogger(__name__) + + +def strtobool(s: str) -> bool: + if s in ['y', 'yes', 'true', 'on', '1']: + return True + if s in ['n', 'no', 'false', 'off', '0']: + return False + + raise ValueError(f'cannot convert {s} to bool') diff --git a/little_boxes/activitypub.py b/little_boxes/activitypub.py index 8205a26..24a1543 100644 --- a/little_boxes/activitypub.py +++ b/little_boxes/activitypub.py @@ -9,7 +9,7 @@ from enum import Enum from .errors import BadActivityError from .errors import UnexpectedActivityTypeError from .errors import NotFromOutboxError -from . import utils +from .utils import parse_collection from .remote_object import OBJECT_FETCHER from typing import List @@ -109,6 +109,84 @@ def _get_actor_id(actor: ObjectOrIDType) -> str: return actor +class Actions(object): + def outbox_is_blocked(self, actor_id: str) -> bool: + return False + + def inbox_get_by_iri(self, iri: str) -> 'BaseActivity': + pass + + def inbox_new(self, activity: 'BaseActivity') -> None: + pass + + def activity_url(self, obj_id: str) -> str: + # from the random hex ID + return 'TODO' + + def outbox_new(self, activity: 'BaseActivity') -> None: + pass + + def new_follower(self, actor: 'Person') -> None: + pass + + def undo_new_follower(self, actor: 'Person') -> None: + pass + + def new_following(self, actor: 'Person') -> None: + pass + + def undo_new_following(self, actor: 'Person') -> None: + pass + + def post_to_remote_inbox(self, payload: ObjectType, recp: str) -> None: + pass + + def is_from_outbox(self, activity: 'BaseActivity') -> None: + pass + + def inbox_like(self, activity: 'Like') -> None: + pass + + def inbox_undo_like(self, activity: 'Like') -> None: + pass + + def outbox_like(self, activity: 'Like') -> None: + pass + + def outbox_undo_like(self, activity: 'Lke') -> None: + pass + + def inbox_announce(self, activity: 'Announce') -> None: + pass + + def inbox_undo_announce(self, activity: 'Announce') -> None: + pass + + def outbox_announce(self, activity: 'Announce') -> None: + pass + + def outbox_undo_announce(self, activity: 'Announce') -> None: + pass + + def inbox_delete(self, activity: 'Delete') -> None: + pass + + def outbox_delete(self, activity: 'Delete') -> None: + pass + + def inbox_update(self, activity: 'Update') -> None: + pass + + def outbox_update(self, activity: 'Update') -> None: + pass + + def inbox_create(self, activity: 'Create') -> None: + pass + + def outbox_create(self, activity: 'Create') -> None: + pass + + class _ActivityMeta(type): """Metaclass for keeping track of subclass.""" def __new__(meta, name, bases, class_dict): @@ -119,16 +197,16 @@ class _ActivityMeta(type): raise ValueError(f'class {name} has no ACTIVITY_TYPE') # Register it - _REGISTER[cls.ACTIVITY_TYPE] = cls + _ACTIVITY_CLS[cls.ACTIVITY_TYPE] = cls return cls -class _BaseActivity(object, metaclass=_ActivityMeta): +class BaseActivity(object, metaclass=_ActivityMeta): """Base class for ActivityPub activities.""" ACTIVITY_TYPE: Optional[ActivityType] = None # the ActivityTypeEnum the class will represent OBJECT_REQUIRED = False # Whether the object field is required or note - ALLOWED_OBJECT_TYPES: List[ActivityType] = [] # + ALLOWED_OBJECT_TYPES: List[ActivityType] = [] ACTOR_REQUIRED = True # Most of the object requires an actor, so this flag in on by default def __init__(self, **kwargs) -> None: @@ -139,7 +217,7 @@ class _BaseActivity(object, metaclass=_ActivityMeta): self._data: Dict[str, Any] = { 'type': self.ACTIVITY_TYPE.value } - logger.debug(f'initializing a {self.ACTIVITY_TYPE.value} activity: {kwargs}') + logger.debug(f'initializing a {self.ACTIVITY_TYPE.value} activity: {kwargs!r}') # The id may not be present for new activities if 'id' in kwargs: @@ -172,7 +250,7 @@ class _BaseActivity(object, metaclass=_ActivityMeta): self._data['object'] = obj if '@context' not in kwargs: - self._data['@context'] = CTX_AS + self._data['@context'] = CTX_AS else: self._data['@context'] = kwargs.pop('@context') @@ -187,7 +265,7 @@ class _BaseActivity(object, metaclass=_ActivityMeta): else: self._data['@context'].append({'Hashtag': 'as:Hashtag', 'sensitive': 'as:sensitive'}) - # FIXME(tsileo): keys required for all subclasses? + # FIXME(tsileo): keys required for some subclasses? allowed_keys = None try: allowed_keys = self._init(**kwargs) @@ -257,7 +335,7 @@ class _BaseActivity(object, metaclass=_ActivityMeta): except Exception: raise BadActivityError(f'failed to validate actor {obj!r}') - if not actor or not 'id' in actor: + if not actor or 'id' not in actor: raise BadActivityError(f'invalid actor {actor}') return actor['id'] @@ -274,7 +352,7 @@ class _BaseActivity(object, metaclass=_ActivityMeta): raise UnexpectedActivityTypeError(f'invalid object type {obj.get("type")!r}') p = parse_activity(obj) - self.__obj: Optional[BaseActivity] = p + self.__obj: Optional['BaseActivity'] = p return p def reset_object_cache(self) -> None: @@ -326,13 +404,6 @@ class _BaseActivity(object, metaclass=_ActivityMeta): def _undo_inbox(self) -> None: raise NotImplementedError - # FIXME(tsileo): delete these? - def _undo_should_purge_cache(self) -> bool: - raise NotImplementedError - - def _should_purge_cache(self) -> bool: - raise NotImplementedError - def process_from_inbox(self) -> None: logger.debug(f'calling main process from inbox hook for {self}') actor = self.get_actor() @@ -340,12 +411,12 @@ class _BaseActivity(object, metaclass=_ActivityMeta): # Check for Block activity # ABC if self.outbox_is_blocked(actor.id): - # TODO(tsileo): raise ActorBlockedError + # TODO(tsileo): raise ActorBlockedError? logger.info(f'actor {actor!r} is blocked, dropping the received activity {self!r}') return # ABC - if self.inbox_get_by_id(self.id): + if self.inbox_get_by_iri(self.id): # The activity is already in the inbox logger.info(f'received duplicate activity {self}, dropping it') return @@ -357,7 +428,7 @@ class _BaseActivity(object, metaclass=_ActivityMeta): logger.debug('pre process from inbox hook not implemented') # ABC - self.inbox_create(self) + self.inbox_new(self) logger.info('activity {self!r} saved') try: @@ -371,7 +442,8 @@ class _BaseActivity(object, metaclass=_ActivityMeta): # Assign create a random ID obj_id = random_object_id() - self.set_id(f'{ID}/outbox/{obj_id}', obj_id) + # ABC + self.set_id(self.activity_url(obj_id), obj_id) try: self._pre_post_to_outbox() @@ -380,11 +452,11 @@ class _BaseActivity(object, metaclass=_ActivityMeta): logger.debug('pre post to outbox hook not implemented') # ABC - self.outbox_create(self) + self.outbox_new(self) recipients = self.recipients() logger.info(f'recipients={recipients}') - activity = clean_activity(activity) + activity = clean_activity(self.to_dict()) try: self._post_to_outbox(obj_id, activity, recipients) @@ -399,9 +471,6 @@ class _BaseActivity(object, metaclass=_ActivityMeta): # ABC self.post_to_remote_inbox(payload, recp) - def _post_to_inbox(self, payload: str, to: str): - tasks.post_to_inbox.delay(payload, to) - def _recipients(self) -> List[str]: return [] @@ -423,7 +492,7 @@ class _BaseActivity(object, metaclass=_ActivityMeta): actor = recipient else: raw_actor = OBJECT_FETCHER.fetch(recipient) - if raw_actor['type'] == ActiivtyType.PERSON.name: + if raw_actor['type'] == ActivityType.PERSON.name: actor = Person(**raw_actor) if actor.endpoints: @@ -437,14 +506,14 @@ class _BaseActivity(object, metaclass=_ActivityMeta): # Is the activity a `Collection`/`OrderedCollection`? elif raw_actor['type'] in [ActivityType.COLLECTION.value, - ActivityType.ORDERED_COLLECTION.value]: + ActivityType.ORDERED_COLLECTION.value]: for item in parse_collection(raw_actor): - if item in [ME, AS_PUBLIC]: + if item in [actor_id, AS_PUBLIC]: continue try: col_actor = Person(**OBJECT_FETCHER.fetch(item)) - except NotAnActorError: - pass + except UnexpectedActivityTypeError: + logger.exception(f'failed to fetch actor {item!r}') if col_actor.endpoints: shared_inbox = col_actor.endpoints.get('sharedInbox') @@ -503,7 +572,6 @@ class Follow(BaseActivity): OBJECT_REQUIRED = True ACTOR_REQUIRED = True - def _build_reply(self, reply_type: ActivityType) -> BaseActivity: if reply_type == ActivityType.ACCEPT: return Accept( @@ -520,18 +588,18 @@ class Follow(BaseActivity): accept = self.build_accept() accept.post_to_outbox() - remote_actor = self.get_actor().id + remote_actor = self.get_actor() # ABC self.new_follower(remote_actor) def _undo_inbox(self) -> None: # ABC - self.undo_new_follower(self.get_actor().id) + self.undo_new_follower(self.get_actor()) def _undo_outbox(self) -> None: # ABC - self.undo_new_following(self.get_object().id) + self.undo_new_following(self.get_actor()) def build_accept(self) -> BaseActivity: return self._build_reply(ActivityType.ACCEPT) @@ -550,7 +618,8 @@ class Accept(BaseActivity): return [self.get_object().get_actor().id] def _pre_process_from_inbox(self) -> None: - # FIXME(tsileo): ensure the actor match the object actor + # FIXME(tsileo): ensure the actor match the object actor + pass def _process_from_inbox(self) -> None: # ABC @@ -591,9 +660,9 @@ class Undo(BaseActivity): def _pre_post_to_outbox(self) -> None: """Ensures an Undo activity references an activity owned by the instance.""" - obj = self.get_object() - if not obj.id.startswith(ID): - raise NotFromOutboxError(f'object {obj["id"]} is not owned by this instance') + # ABC + if not self.is_from_outbox(self): + raise NotFromOutboxError(f'object {self!r} is not owned by this instance') def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: logger.debug('processing undo to outbox') @@ -622,43 +691,20 @@ class Like(BaseActivity): return [self.get_object().get_actor().id] def _process_from_inbox(self): - obj = self.get_object() - # Update the meta counter if the object is published by the server - # FIXME(tsileo): continue here - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_like': 1}, - }) - # XXX(tsileo): notification?? + # ABC + self.inbox_like(self) def _undo_inbox(self) -> None: - obj = self.get_object() - # Update the meta counter if the object is published by the server - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_like': -1}, - }) - - def _undo_should_purge_cache(self) -> bool: - # If a like coutn was decremented, we need to purge the application cache - return self.get_object().id.startswith(BASE_URL) + # ABC + self.inbox_undo_like(self) def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]): - obj = self.get_object() - # Unlikely, but an actor can like it's own post - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_like': 1}, - }) - - # Keep track of the like we just performed - DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.liked': obj_id}}) + # ABC + self.outbox_like(self) def _undo_outbox(self) -> None: - obj = self.get_object() - # Unlikely, but an actor can like it's own post - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_like': -1}, - }) - - DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.liked': False}}) + # ABC + self.outbox_undo_like(self) def build_undo(self) -> BaseActivity: return Undo(object=self.to_dict(embed=True, embed_object_id_only=True)) @@ -667,6 +713,8 @@ class Like(BaseActivity): class Announce(BaseActivity): ACTIVITY_TYPE = ActivityType.ANNOUNCE ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] + OBJECT_REQUIRED = True + ACTOR_REQUIRED = True def _recipients(self) -> List[str]: recipients = [] @@ -684,49 +732,21 @@ class Announce(BaseActivity): f'received an Annouce referencing an OStatus notice ({self._data["object"]}), dropping the message' ) return - # Save/cache the object, and make it part of the stream so we can fetch it - if isinstance(self._data['object'], str): - raw_obj = OBJECT_SERVICE.get( - self._data['object'], - reload_cache=True, - part_of_stream=True, - announce_published=self._data['published'], - ) - obj = parse_activity(raw_obj) - else: - obj = self.get_object() - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_boost': 1}, - }) + # ABC + self.inbox_announce(self) def _undo_inbox(self) -> None: - obj = self.get_object() - # Update the meta counter if the object is published by the server - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_boost': -1}, - }) - - def _undo_should_purge_cache(self) -> bool: - # If a like coutn was decremented, we need to purge the application cache - return self.get_object().id.startswith(BASE_URL) + # ABC + self.inbox_undo_annnounce(self) def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - if isinstance(self._data['object'], str): - # Put the object in the cache - OBJECT_SERVICE.get( - self._data['object'], - reload_cache=True, - part_of_stream=True, - announce_published=self._data['published'], - ) - - obj = self.get_object() - DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.boosted': obj_id}}) + # ABC + self.outbox_announce(self) def _undo_outbox(self) -> None: - obj = self.get_object() - DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.boosted': False}}) + # ABC + self.outbox_undo_announce(self) def build_undo(self) -> BaseActivity: return Undo(object=self.to_dict(embed=True)) @@ -738,9 +758,10 @@ class Delete(BaseActivity): OBJECT_REQUIRED = True def _get_actual_object(self) -> BaseActivity: + # FIXME(tsileo): overrides get_object instead? obj = self.get_object() if obj.type_enum == ActivityType.TOMBSTONE: - obj = parse_activity(OBJECT_SERVICE.get(obj.id)) + obj = parse_activity(OBJECT_FETCHER.fetch(obj.id)) return obj def _recipients(self) -> List[str]: @@ -755,27 +776,27 @@ class Delete(BaseActivity): raise BadActivityError(f'{actor!r} cannot delete {obj!r}') def _process_from_inbox(self) -> None: - DB.inbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) - obj = self._get_actual_object() - if obj.type_enum == ActivityType.NOTE: - obj._delete_from_threads() - - # TODO(tsileo): also purge the cache if it's a reply of a published activity + # ABC + self.inbox_delete(self) + # FIXME(tsileo): handle the delete_threads here? def _pre_post_to_outbox(self) -> None: """Ensures the Delete activity references a activity from the outbox (i.e. owned by the instance).""" obj = self._get_actual_object() - if not obj.id.startswith(ID): + # ABC + if not self.is_from_outbox(self): raise NotFromOutboxError(f'object {obj["id"]} is not owned by this instance') def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - DB.outbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) + # ABC + self.outbox_delete(self) class Update(BaseActivity): ACTIVITY_TYPE = ActivityType.UPDATE ALLOWED_OBJECT_TYPES = [ActivityType.NOTE, ActivityType.PERSON] OBJECT_REQUIRED = True + ACTOR_REQUIRED = True def _pre_process_from_inbox(self) -> None: """Ensures an Update activity comes from the same actor as the updated activity.""" @@ -785,53 +806,29 @@ class Update(BaseActivity): raise BadActivityError(f'{actor!r} cannot update {obj!r}') def _process_from_inbox(self): - obj = self.get_object() - if obj.type_enum == ActivityType.NOTE: - DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'activity.object': obj.to_dict()}}) - return - - # If the object is a Person, it means the profile was updated, we just refresh our local cache - ACTOR_SERVICE.get(obj.id, reload_cache=True) - - # TODO(tsileo): implements _should_purge_cache if it's a reply of a published activity (i.e. in the outbox) + # ABC + self.inbox_update(self) def _pre_post_to_outbox(self) -> None: - obj = self.get_object() - if not obj.id.startswith(ID): - raise NotFromOutboxError(f'object {obj["id"]} is not owned by this instance') + # ABC + if not self.is_form_outbox(self): + raise NotFromOutboxError(f'object {self!r} is not owned by this instance') def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - obj = self._data['object'] - - update_prefix = 'activity.object.' - update: Dict[str, Any] = {'$set': dict(), '$unset': dict()} - update['$set'][f'{update_prefix}updated'] = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' - for k, v in obj.items(): - if k in ['id', 'type']: - continue - if v is None: - update['$unset'][f'{update_prefix}{k}'] = '' - else: - update['$set'][f'{update_prefix}{k}'] = v - - if len(update['$unset']) == 0: - del(update['$unset']) - - print(f'updating note from outbox {obj!r} {update}') - logger.info(f'updating note from outbox {obj!r} {update}') - DB.outbox.update_one({'activity.object.id': obj['id']}, update) - # FIXME(tsileo): should send an Update (but not a partial one, to all the note's recipients - # (create a new Update with the result of the update, and send it without saving it?) + # ABC + self.outbox_update(self) class Create(BaseActivity): ACTIVITY_TYPE = ActivityType.CREATE ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] OBJECT_REQUIRED = True + ACTOR_REQUIRED = True def _set_id(self, uri: str, obj_id: str) -> None: self._data['object']['id'] = uri + '/activity' - self._data['object']['url'] = ID + '/' + self.get_object().type.lower() + '/' + obj_id + # ABC + self._data['object']['url'] = self.note_url(self) self.reset_object_cache() def _init(self, **kwargs): @@ -857,83 +854,25 @@ class Create(BaseActivity): return recipients - def _update_threads(self) -> None: - logger.debug('_update_threads hook') - obj = self.get_object() - - # TODO(tsileo): re-enable me - # tasks.fetch_og.delay('INBOX', self.id) - - threads = [] - reply = obj.get_local_reply() - print(f'initial_reply={reply}') - print(f'{obj}') - logger.debug(f'initial_reply={reply}') - reply_id = None - direct_reply = 1 - while reply is not None: - if not DB.inbox.find_one_and_update({'activity.object.id': reply.id}, { - '$inc': { - 'meta.count_reply': 1, - 'meta.count_direct_reply': direct_reply, - }, - '$addToSet': {'meta.thread_children': obj.id}, - }): - DB.outbox.update_one({'activity.object.id': reply.id}, { - '$inc': { - 'meta.count_reply': 1, - 'meta.count_direct_reply': direct_reply, - }, - '$addToSet': {'meta.thread_children': obj.id}, - }) - - direct_reply = 0 - reply_id = reply.id - reply = reply.get_local_reply() - logger.debug(f'next_reply={reply}') - threads.append(reply_id) - # FIXME(tsileo): obj.id is None!! - print(f'reply_id={reply_id} {obj.id} {obj._data} {self.id}') - - if reply_id: - if not DB.inbox.find_one_and_update({'activity.object.id': obj.id}, { - '$set': { - 'meta.thread_parents': threads, - 'meta.thread_root_parent': reply_id, - }, - }): - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$set': { - 'meta.thread_parents': threads, - 'meta.thread_root_parent': reply_id, - }, - }) - logger.debug('_update_threads done') - def _process_from_inbox(self) -> None: - self._update_threads() + # ABC + self.inbox_create(self) def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - self._update_threads() - - def _should_purge_cache(self) -> bool: - # TODO(tsileo): handle reply of a reply... - obj = self.get_object() - in_reply_to = obj.inReplyTo - if in_reply_to: - local_activity = DB.outbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) - if local_activity: - return True - - return False + # ABC + self.outbox_create(self) class Tombstone(BaseActivity): ACTIVITY_TYPE = ActivityType.TOMBSTONE + ACTOR_REQUIRED = False + OBJECT_REQUIRED = False class Note(BaseActivity): ACTIVITY_TYPE = ActivityType.NOTE + ACTOR_REQUIRED = True + OBJECT_REQURIED = False def _init(self, **kwargs): print(self._data) @@ -947,11 +886,12 @@ class Note(BaseActivity): # TODO(tsileo): audience support? recipients: List[str] = [] + # FIXME(tsileo): re-add support for the PUBLIC_INSTANCES # If the note is public, we publish it to the defined "public instances" - if AS_PUBLIC in self._data.get('to', []): - recipients.extend(PUBLIC_INSTANCES) - print('publishing to public instances') - print(recipients) + # if AS_PUBLIC in self._data.get('to', []): + # recipients.extend(PUBLIC_INSTANCES) + # print('publishing to public instances') + # print(recipients) for field in ['to', 'cc', 'bto', 'bcc']: if field in self._data: @@ -959,59 +899,11 @@ class Note(BaseActivity): return recipients - def _delete_from_threads(self) -> None: - logger.debug('_delete_from_threads hook') - - reply = self.get_local_reply() - logger.debug(f'initial_reply={reply}') - direct_reply = -1 - while reply is not None: - if not DB.inbox.find_one_and_update({'activity.object.id': reply.id}, { - '$inc': { - 'meta.count_reply': -1, - 'meta.count_direct_reply': direct_reply, - }, - '$pull': {'meta.thread_children': self.id}, - - }): - DB.outbox.update_one({'activity.object.id': reply.id}, { - '$inc': { - 'meta.count_reply': 1, - 'meta.count_direct_reply': direct_reply, - }, - '$pull': {'meta.thread_children': self.id}, - }) - - direct_reply = 0 - reply = reply.get_local_reply() - logger.debug(f'next_reply={reply}') - - logger.debug('_delete_from_threads done') - return None - - def get_local_reply(self) -> Optional[BaseActivity]: - "Find the note reply if any.""" - in_reply_to = self.inReplyTo - if not in_reply_to: - # This is the root comment - return None - - inbox_parent = DB.inbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) - if inbox_parent: - return parse_activity(inbox_parent['activity']['object']) - - outbox_parent = DB.outbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) - if outbox_parent: - return parse_activity(outbox_parent['activity']['object']) - - # The parent is no stored on this instance - return None - def build_create(self) -> BaseActivity: """Wraps an activity in a Create activity.""" create_payload = { 'object': self.to_dict(embed=True), - 'actor': self.attributedTo or ME, + 'actor': self.attributedTo, } for field in ['published', 'to', 'bto', 'cc', 'bcc', 'audience']: if field in self._data: @@ -1026,7 +918,7 @@ class Note(BaseActivity): return Announce( object=self.id, to=[AS_PUBLIC], - cc=[ID+'/followers', self.attributedTo], + cc=[self.follower_collection_id(self.get_actor()), self.attributedTo], # ABC published=datetime.utcnow().replace(microsecond=0).isoformat() + 'Z', ) diff --git a/little_boxes/errors.py b/little_boxes/errors.py index f70159a..cb3cb34 100644 --- a/little_boxes/errors.py +++ b/little_boxes/errors.py @@ -8,7 +8,11 @@ class Error(Exception): """HTTP-friendly base error, with a status code, a message and an optional payload.""" status_code = 400 - def __init__(self, message: str, status_code: Optional[int] = None, payload: Optional[Dict[str, Any]] = None) -> None: + def __init__( + self, message: str, + status_code: Optional[int] = None, + payload: Optional[Dict[str, Any]] = None, + ) -> None: Exception.__init__(self) self.message = message if status_code is not None: @@ -21,7 +25,9 @@ class Error(Exception): return rv def __repr__(self) -> str: - return f'{self.__class__.__qualname__}({self.message!r}, payload={self.payload!r}, status_code={self.status_code})' + return ( + f'{self.__class__.__qualname__}({self.message!r}, payload={self.payload!r}, status_code={self.status_code})' + ) class ActorBlockedError(Error): diff --git a/little_boxes/remote_object.py b/little_boxes/remote_object.py index 9a528bf..f200609 100644 --- a/little_boxes/remote_object.py +++ b/little_boxes/remote_object.py @@ -2,12 +2,9 @@ import logging from typing import Any import requests -from urllib.parse import urlparse -from Crypto.PublicKey import RSA from .urlutils import check_url from .errors import ActivityNotFoundError -from .errors import UnexpectedActivityTypeError logger = logging.getLogger(__name__) @@ -21,19 +18,22 @@ class DefaultRemoteObjectFetcher(object): def fetch(self, iri): check_url(iri) - resp = requests.get(actor_url, headers={ + resp = requests.get(iri, headers={ 'Accept': 'application/activity+json', - 'User-Agent': self._user_agent, + 'User-Agent': self._user_agent, }) if resp.status_code == 404: - raise ActivityNotFoundError(f'{actor_url} cannot be fetched, 404 not found error') + raise ActivityNotFoundError(f'{iri} cannot be fetched, 404 not found error') resp.raise_for_status() - + return resp.json() + OBJECT_FETCHER = DefaultRemoteObjectFetcher() + def set_object_fetcher(object_fetcher: Any): + global OBJECT_FETCHER OBJECT_FETCHER = object_fetcher diff --git a/little_boxes/utils.py b/little_boxes/utils.py index 64987c2..aef77f4 100644 --- a/little_boxes/utils.py +++ b/little_boxes/utils.py @@ -4,7 +4,6 @@ from typing import Dict from typing import List from typing import Any -import requests from .errors import RecursionLimitExceededError from .errors import UnexpectedActivityTypeError @@ -21,10 +20,6 @@ def parse_collection( raise RecursionLimitExceededError('recursion limit exceeded') # Go through all the pages - headers = {'Accept': 'application/activity+json'} - if user_agent: - headers['User-Agent'] = user_agent - out: List[Any] = [] if url: payload = OBJECT_FETCHER.fetch(url) From f0880f011962cf7aa9eb0bcffaff04b24b496eff Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 9 Jun 2018 10:15:52 +0200 Subject: [PATCH 0107/1425] [WIP] More progress --- little_boxes/README.md | 41 ++++++++++++++++++++++++ little_boxes/activitypub.py | 64 +++++++++++++++++++++++++++++-------- 2 files changed, 91 insertions(+), 14 deletions(-) create mode 100644 little_boxes/README.md diff --git a/little_boxes/README.md b/little_boxes/README.md new file mode 100644 index 0000000..fb491af --- /dev/null +++ b/little_boxes/README.md @@ -0,0 +1,41 @@ +# Little Boxes + +Tiny ActivityPub framework. + +## Getting Started + +```python +from little_boxes.activitypub import BaseBackend +from little_boxes.activitypub import use_backend +from little_boxes.activitypub import Outbox +from little_boxes.activitypub import Person +from little_boxes.activitypub import Follow + +from mydb import db_client + + +class MyBackend(BaseBackend): + + def __init__(self, db_connection): + self.db_connection = db_connection + + def inbox_new(self, as_actor, activity): + # Save activity as "as_actor" + # [...] + + def post_to_remote_inbox(self, as_actor, payload, recipient): + # Send the activity to the remote actor + # [...] + + +db_con = db_client() +my_backend = MyBackend(db_con) + +use_backend(my_backend) + +me = Person({}) # Init an actor +outbox = Outbox(me) + +follow = Follow(actor=me, object='http://iri-i-want-follow') +outbox.post(follow) +``` diff --git a/little_boxes/activitypub.py b/little_boxes/activitypub.py index 24a1543..6e992dd 100644 --- a/little_boxes/activitypub.py +++ b/little_boxes/activitypub.py @@ -41,6 +41,12 @@ COLLECTION_CTX = [ # Will be used to keep track of all the defined activities _ACTIVITY_CLS: Dict['ActivityTypeEnum', Type['_BaseActivity']] = {} +BACKEND = None + +def use_backend(backend_instance): + global BACKEND + BACKEND = backend_instance + class ActivityType(Enum): """Supported activity `type`.""" @@ -109,14 +115,16 @@ def _get_actor_id(actor: ObjectOrIDType) -> str: return actor -class Actions(object): - def outbox_is_blocked(self, actor_id: str) -> bool: - return False +class BaseBackend(object): - def inbox_get_by_iri(self, iri: str) -> 'BaseActivity': + def outbox_is_blocked(self, as_actor: 'Person', actor_id: str) -> bool: + """Returns True if `as_actor` has blocked `actor_id`.""" pass - def inbox_new(self, activity: 'BaseActivity') -> None: + def inbox_get_by_iri(self, as_actor: 'Person', iri: str) -> 'BaseActivity': + pass + + def inbox_new(self, as_actor: 'Person', activity: 'BaseActivity') -> None: pass def activity_url(self, obj_id: str) -> str: @@ -404,31 +412,29 @@ class BaseActivity(object, metaclass=_ActivityMeta): def _undo_inbox(self) -> None: raise NotImplementedError - def process_from_inbox(self) -> None: + def process_from_inbox(self, as_actor: 'Person') -> None: + """Process the message posted to `as_actor` inbox.""" logger.debug(f'calling main process from inbox hook for {self}') actor = self.get_actor() # Check for Block activity - # ABC - if self.outbox_is_blocked(actor.id): + if BACKEND.outbox_is_blocked(as_actor, actor.id): # TODO(tsileo): raise ActorBlockedError? logger.info(f'actor {actor!r} is blocked, dropping the received activity {self!r}') return - # ABC - if self.inbox_get_by_iri(self.id): + if BACKEND.inbox_get_by_iri(as_actor, self.id): # The activity is already in the inbox logger.info(f'received duplicate activity {self}, dropping it') return try: - self._pre_process_from_inbox() + self._pre_process_from_inbox(as_actor) logger.debug('called pre process from inbox hook') except NotImplementedError: logger.debug('pre process from inbox hook not implemented') - # ABC - self.inbox_new(self) + BACKEND.inbox_new(as_actor, self) logger.info('activity {self!r} saved') try: @@ -707,7 +713,10 @@ class Like(BaseActivity): self.outbox_undo_like(self) def build_undo(self) -> BaseActivity: - return Undo(object=self.to_dict(embed=True, embed_object_id_only=True)) + return Undo( + object=self.to_dict(embed=True, embed_object_id_only=True), + actor=self.get_actor().id, + ) class Announce(BaseActivity): @@ -932,3 +941,30 @@ class Note(BaseActivity): deleted=deleted, updated=deleted, ) + + +class Box(object): + def __init__(self, actor: Person): + self.actor = actor + + +class Outbox(Box): + + def post(self, activity: BaseActivity) -> None: + if activity.get_actor().id != self.actor.id: + raise ValueError(f'{activity.get_actor()!r} cannot post into {self.actor!r} outbox') + + activity.post_to_outbox() + + def get(self, activity_iri: str) -> BaseActivity: + pass + + def collection(self): + # TODO(tsileo): figure out an API + + +class Inbox(Box): + + def post(self, activity: BaseActivity) -> None: + + activity.process_from_inbox(self.actor) From b75da521e4eae697c12361f501a2bd1226c12d20 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 10 Jun 2018 13:51:43 +0200 Subject: [PATCH 0108/1425] [WIP] Start an in-mem backend --- little_boxes/README.md | 2 +- little_boxes/activitypub.py | 135 ++++++++++++++++++++++++++++------ little_boxes/remote_object.py | 39 ---------- little_boxes/utils.py | 11 ++- 4 files changed, 119 insertions(+), 68 deletions(-) delete mode 100644 little_boxes/remote_object.py diff --git a/little_boxes/README.md b/little_boxes/README.md index fb491af..021a0f2 100644 --- a/little_boxes/README.md +++ b/little_boxes/README.md @@ -1,6 +1,6 @@ # Little Boxes -Tiny ActivityPub framework. +Tiny ActivityPub framework written in Python, both database and server agnostic. ## Getting Started diff --git a/little_boxes/activitypub.py b/little_boxes/activitypub.py index 6e992dd..c6dc9a0 100644 --- a/little_boxes/activitypub.py +++ b/little_boxes/activitypub.py @@ -9,8 +9,9 @@ from enum import Enum from .errors import BadActivityError from .errors import UnexpectedActivityTypeError from .errors import NotFromOutboxError +from .errors import ActivityNotFoundError +from .urlutils import check_url from .utils import parse_collection -from .remote_object import OBJECT_FETCHER from typing import List from typing import Optional @@ -19,6 +20,9 @@ from typing import Any from typing import Union from typing import Type +import requests + + logger = logging.getLogger(__name__) # Helper/shortcut for typing @@ -48,6 +52,30 @@ def use_backend(backend_instance): BACKEND = backend_instance + +class DefaultRemoteObjectFetcher(object): + """Not meant to be used on production, a caching layer, and DB shortcut fox inbox/outbox should be hooked.""" + + def __init__(self): + self._user_agent = 'Little Boxes (+https://github.com/tsileo/little_boxes)' + + def fetch(self, iri): + print('OLD FETCHER') + check_url(iri) + + resp = requests.get(iri, headers={ + 'Accept': 'application/activity+json', + 'User-Agent': self._user_agent, + }) + + if resp.status_code == 404: + raise ActivityNotFoundError(f'{iri} cannot be fetched, 404 not found error') + + resp.raise_for_status() + + return resp.json() + + class ActivityType(Enum): """Supported activity `type`.""" ANNOUNCE = 'Announce' @@ -116,38 +144,95 @@ def _get_actor_id(actor: ObjectOrIDType) -> str: class BaseBackend(object): + """In-memory backend meant to be used for the test suite.""" + DB = {} + USERS = {} + FETCH_MOCK = {} + INBOX_IDX = {} + OUTBOX_IDX = {} + FOLLOWERS = {} + FOLLOWING = {} + + def __init__(self): + self._setup_user('Thomas2', 'tom2') + self._setup_user('Thomas', 'tom') + + def _setup_user(self, name, pusername): + p = Person( + name=name, + preferredUsername=pusername, + summary='Hello', + id=f'https://lol.com/{pusername}', + inbox=f'https://lol.com/{pusername}/inbox', + ) + + self.USERS[p.preferredUsername] = p + self.DB[p.id] = { + 'inbox': [], + 'outbox': [], + } + self.INBOX_IDX[p.id] = {} + self.OUTBOX_IDX[p.id] = {} + self.FOLLOWERS[p.id] = [] + self.FOLLOWING[p.id] = [] + self.FETCH_MOCK[p.id] = p.to_dict() + + def fetch_iri(self, iri: str): + return self.FETCH_MOCK[iri] + + def get_user(self, username: str) -> 'Person': + if username in self.USERS: + return self.USERS[username] + else: + raise ValueError(f'bad username {username}') def outbox_is_blocked(self, as_actor: 'Person', actor_id: str) -> bool: """Returns True if `as_actor` has blocked `actor_id`.""" - pass + for activity in self.DB[as_actor.id]['outbox']: + if activity.ACTIVITY_TYPE == ActivityType.BLOCK: + return True + return False def inbox_get_by_iri(self, as_actor: 'Person', iri: str) -> 'BaseActivity': - pass + for activity in self.DB[as_actor.id]['inbox']: + if activity.id == iri: + return activity + + raise ActivityNotFoundError() def inbox_new(self, as_actor: 'Person', activity: 'BaseActivity') -> None: - pass + if activity.id in self.INBOX_IDX[as_actor.id]: + return + self.DB[as_actor.id]['inbox'].append(activity) + self.INBOX_IDX[as_actor.id][activity.id] = activity def activity_url(self, obj_id: str) -> str: # from the random hex ID return 'TODO' def outbox_new(self, activity: 'BaseActivity') -> None: - pass + print(f'saving {activity!r} to DB') + actor_id = activity.get_actor().id + if activity.id in self.OUTBOX_IDX[actor_id]: + return + self.DB[actor_id]['outbox'].append(activity) + self.OUTBOX_IDX[actor_id][activity.id] = activity - def new_follower(self, actor: 'Person') -> None: + def new_follower(self, as_actor: 'Person', actor: 'Person') -> None: pass def undo_new_follower(self, actor: 'Person') -> None: pass - def new_following(self, actor: 'Person') -> None: - pass + def new_following(self, as_actor: 'Person', actor: 'Person') -> None: + print(f'new following {actor!r}') + self.FOLLOWING[as_actor.id].append(actor) def undo_new_following(self, actor: 'Person') -> None: pass def post_to_remote_inbox(self, payload: ObjectType, recp: str) -> None: - pass + print(f'post_to_remote_inbox {payload} {recp}') def is_from_outbox(self, activity: 'BaseActivity') -> None: pass @@ -201,7 +286,7 @@ class _ActivityMeta(type): cls = type.__new__(meta, name, bases, class_dict) # Ensure the class has an activity type defined - if not cls.ACTIVITY_TYPE: + if name != 'BaseActivity' and not cls.ACTIVITY_TYPE: raise ValueError(f'class {name} has no ACTIVITY_TYPE') # Register it @@ -321,7 +406,7 @@ class BaseActivity(object, metaclass=_ActivityMeta): logger.debug(f'setting ID {uri} / {obj_id}') self._data['id'] = uri try: - self._set_id(uri, obj_id) + self._outbox_set_id(uri, obj_id) except NotImplementedError: pass @@ -339,7 +424,7 @@ class BaseActivity(object, metaclass=_ActivityMeta): def _validate_person(self, obj: ObjectOrIDType) -> str: obj_id = self._actor_id(obj) try: - actor = OBJECT_FETCHER.fetch(obj_id) + actor = BACKEND.fetch_iri(obj_id) except Exception: raise BadActivityError(f'failed to validate actor {obj!r}') @@ -355,10 +440,10 @@ class BaseActivity(object, metaclass=_ActivityMeta): if isinstance(self._data['object'], dict): p = parse_activity(self._data['object']) else: - obj = OBJECT_FETCHER.fetch(self._data['object']) + obj = BACKEND.fetch_iri(self._data['object']) if ActivityType(obj.get('type')) not in self.ALLOWED_OBJECT_TYPES: raise UnexpectedActivityTypeError(f'invalid object type {obj.get("type")!r}') - p = parse_activity(obj) + p = parse_activity(obj) self.__obj: Optional['BaseActivity'] = p return p @@ -392,7 +477,7 @@ class BaseActivity(object, metaclass=_ActivityMeta): raise BadActivityError(f'failed to fetch actor: {self._data!r}') actor_id = self._actor_id(actor) - return Person(**OBJECT_FETCHER.fetch(actor_id)) + return Person(**BACKEND.fetch_iri(actor_id)) def _pre_post_to_outbox(self) -> None: raise NotImplementedError @@ -449,7 +534,7 @@ class BaseActivity(object, metaclass=_ActivityMeta): # Assign create a random ID obj_id = random_object_id() # ABC - self.set_id(self.activity_url(obj_id), obj_id) + self.outbox_set_id(BACKEND.activity_url(obj_id), obj_id) try: self._pre_post_to_outbox() @@ -457,8 +542,7 @@ class BaseActivity(object, metaclass=_ActivityMeta): except NotImplementedError: logger.debug('pre post to outbox hook not implemented') - # ABC - self.outbox_new(self) + BACKEND.outbox_new(self) recipients = self.recipients() logger.info(f'recipients={recipients}') @@ -475,7 +559,7 @@ class BaseActivity(object, metaclass=_ActivityMeta): logger.debug(f'posting to {recp}') # ABC - self.post_to_remote_inbox(payload, recp) + BACKEND.post_to_remote_inbox(payload, recp) def _recipients(self) -> List[str]: return [] @@ -497,8 +581,8 @@ class BaseActivity(object, metaclass=_ActivityMeta): continue actor = recipient else: - raw_actor = OBJECT_FETCHER.fetch(recipient) - if raw_actor['type'] == ActivityType.PERSON.name: + raw_actor = BACKEND.fetch_iri(recipient) + if raw_actor['type'] == ActivityType.PERSON.value: actor = Person(**raw_actor) if actor.endpoints: @@ -517,7 +601,7 @@ class BaseActivity(object, metaclass=_ActivityMeta): if item in [actor_id, AS_PUBLIC]: continue try: - col_actor = Person(**OBJECT_FETCHER.fetch(item)) + col_actor = Person(**BACKEND.fetch_iri(item)) except UnexpectedActivityTypeError: logger.exception(f'failed to fetch actor {item!r}') @@ -599,6 +683,9 @@ class Follow(BaseActivity): # ABC self.new_follower(remote_actor) + def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: + BACKEND.new_following(self.get_actor(), self.get_object()) + def _undo_inbox(self) -> None: # ABC self.undo_new_follower(self.get_actor()) @@ -770,7 +857,7 @@ class Delete(BaseActivity): # FIXME(tsileo): overrides get_object instead? obj = self.get_object() if obj.type_enum == ActivityType.TOMBSTONE: - obj = parse_activity(OBJECT_FETCHER.fetch(obj.id)) + obj = parse_activity(BACKEND.fetch_iri(obj.id)) return obj def _recipients(self) -> List[str]: @@ -961,10 +1048,10 @@ class Outbox(Box): def collection(self): # TODO(tsileo): figure out an API + pass class Inbox(Box): def post(self, activity: BaseActivity) -> None: - activity.process_from_inbox(self.actor) diff --git a/little_boxes/remote_object.py b/little_boxes/remote_object.py deleted file mode 100644 index f200609..0000000 --- a/little_boxes/remote_object.py +++ /dev/null @@ -1,39 +0,0 @@ -import logging -from typing import Any - -import requests - -from .urlutils import check_url -from .errors import ActivityNotFoundError - -logger = logging.getLogger(__name__) - - -class DefaultRemoteObjectFetcher(object): - """Not meant to be used on production, a caching layer, and DB shortcut fox inbox/outbox should be hooked.""" - - def __init__(self): - self._user_agent = 'Little Boxes (+https://github.com/tsileo/little_boxes)' - - def fetch(self, iri): - check_url(iri) - - resp = requests.get(iri, headers={ - 'Accept': 'application/activity+json', - 'User-Agent': self._user_agent, - }) - - if resp.status_code == 404: - raise ActivityNotFoundError(f'{iri} cannot be fetched, 404 not found error') - - resp.raise_for_status() - - return resp.json() - - -OBJECT_FETCHER = DefaultRemoteObjectFetcher() - - -def set_object_fetcher(object_fetcher: Any): - global OBJECT_FETCHER - OBJECT_FETCHER = object_fetcher diff --git a/little_boxes/utils.py b/little_boxes/utils.py index aef77f4..2476182 100644 --- a/little_boxes/utils.py +++ b/little_boxes/utils.py @@ -1,5 +1,6 @@ """Contains some ActivityPub related utils.""" from typing import Optional +from typing import Callable from typing import Dict from typing import List from typing import Any @@ -7,22 +8,24 @@ from typing import Any from .errors import RecursionLimitExceededError from .errors import UnexpectedActivityTypeError -from .remote_object import OBJECT_FETCHER def parse_collection( payload: Optional[Dict[str, Any]] = None, url: Optional[str] = None, level: int = 0, + fetcher: Optional[Callable[[str], Dict[str, Any]]] = None, ) -> List[Any]: """Resolve/fetch a `Collection`/`OrderedCollection`.""" + if not fetcher: + raise Exception('must provide a fetcher') if level > 3: raise RecursionLimitExceededError('recursion limit exceeded') # Go through all the pages out: List[Any] = [] if url: - payload = OBJECT_FETCHER.fetch(url) + payload = fetcher(url) if not payload: raise ValueError('must at least prove a payload or an URL') @@ -38,7 +41,7 @@ def parse_collection( out.extend(payload['first']['items']) n = payload['first'].get('next') if n: - out.extend(parse_collection(url=n, level=level+1)) + out.extend(parse_collection(url=n, level=level+1, fetcher=fetcher)) return out while payload: @@ -50,7 +53,7 @@ def parse_collection( n = payload.get('next') if n is None: break - payload = OBJECT_FETCHER.fetch(n) + payload = fetcher(n) else: raise UnexpectedActivityTypeError('unexpected activity type {}'.format(payload['type'])) From b0cf350ec63f155d98bb7a56fcbf6a0eb53cf5de Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 10 Jun 2018 16:58:25 +0200 Subject: [PATCH 0109/1425] [WIP] Start tests for little_boxes --- little_boxes/activitypub.py | 78 ++++++++++++++++++++++--------------- test_little_boxes.py | 26 +++++++++++++ 2 files changed, 73 insertions(+), 31 deletions(-) create mode 100644 test_little_boxes.py diff --git a/little_boxes/activitypub.py b/little_boxes/activitypub.py index c6dc9a0..781c71e 100644 --- a/little_boxes/activitypub.py +++ b/little_boxes/activitypub.py @@ -153,11 +153,8 @@ class BaseBackend(object): FOLLOWERS = {} FOLLOWING = {} - def __init__(self): - self._setup_user('Thomas2', 'tom2') - self._setup_user('Thomas', 'tom') - - def _setup_user(self, name, pusername): + def setup_actor(self, name, pusername): + """Create a new actor in this backend.""" p = Person( name=name, preferredUsername=pusername, @@ -176,6 +173,7 @@ class BaseBackend(object): self.FOLLOWERS[p.id] = [] self.FOLLOWING[p.id] = [] self.FETCH_MOCK[p.id] = p.to_dict() + return p def fetch_iri(self, iri: str): return self.FETCH_MOCK[iri] @@ -193,12 +191,12 @@ class BaseBackend(object): return True return False - def inbox_get_by_iri(self, as_actor: 'Person', iri: str) -> 'BaseActivity': + def inbox_get_by_iri(self, as_actor: 'Person', iri: str) -> Optional['BaseActivity']: for activity in self.DB[as_actor.id]['inbox']: if activity.id == iri: return activity - raise ActivityNotFoundError() + return None def inbox_new(self, as_actor: 'Person', activity: 'BaseActivity') -> None: if activity.id in self.INBOX_IDX[as_actor.id]: @@ -219,20 +217,30 @@ class BaseBackend(object): self.OUTBOX_IDX[actor_id][activity.id] = activity def new_follower(self, as_actor: 'Person', actor: 'Person') -> None: - pass + self.FOLLOWERS[as_actor.id].append(actor.id) def undo_new_follower(self, actor: 'Person') -> None: pass def new_following(self, as_actor: 'Person', actor: 'Person') -> None: print(f'new following {actor!r}') - self.FOLLOWING[as_actor.id].append(actor) + self.FOLLOWING[as_actor.id].append(actor.id) def undo_new_following(self, actor: 'Person') -> None: pass - def post_to_remote_inbox(self, payload: ObjectType, recp: str) -> None: + def followers(self, as_actor: 'Person') -> List[str]: + return self.FOLLOWERS[as_actor.id] + + def following(self, as_actor: 'Person') -> List[str]: + return self.FOLLOWING[as_actor.id] + + def post_to_remote_inbox(self, payload_encoded: str, recp: str) -> None: + payload = json.loads(payload_encoded) print(f'post_to_remote_inbox {payload} {recp}') + act = parse_activity(payload) + as_actor = parse_activity(self.fetch_iri(recp.replace('/inbox', ''))) + act.process_from_inbox(as_actor) def is_from_outbox(self, activity: 'BaseActivity') -> None: pass @@ -312,6 +320,9 @@ class BaseActivity(object, metaclass=_ActivityMeta): } logger.debug(f'initializing a {self.ACTIVITY_TYPE.value} activity: {kwargs!r}') + # A place to set ephemeral data + self.__ctx = {} + # The id may not be present for new activities if 'id' in kwargs: self._data['id'] = kwargs.pop('id') @@ -380,6 +391,12 @@ class BaseActivity(object, metaclass=_ActivityMeta): valid_kwargs[k] = v self._data.update(**valid_kwargs) + def ctx(self) -> Dict[str, Any]: + return self.__ctx + + def set_ctx(self, ctx: Dict[str, Any]) -> None: + self.__ctx = ctx + def _init(self, **kwargs) -> Optional[List[str]]: """Optional init callback that may returns a list of allowed keys.""" raise NotImplementedError @@ -488,10 +505,10 @@ class BaseActivity(object, metaclass=_ActivityMeta): def _undo_outbox(self) -> None: raise NotImplementedError - def _pre_process_from_inbox(self) -> None: + def _pre_process_from_inbox(self, as_actor: 'Person') -> None: raise NotImplementedError - def _process_from_inbox(self) -> None: + def _process_from_inbox(self, as_actor: 'Person') -> None: raise NotImplementedError def _undo_inbox(self) -> None: @@ -523,7 +540,7 @@ class BaseActivity(object, metaclass=_ActivityMeta): logger.info('activity {self!r} saved') try: - self._process_from_inbox() + self._process_from_inbox(as_actor) logger.debug('called process from inbox hook') except NotImplementedError: logger.debug('process from inbox hook not implemented') @@ -533,7 +550,6 @@ class BaseActivity(object, metaclass=_ActivityMeta): # Assign create a random ID obj_id = random_object_id() - # ABC self.outbox_set_id(BACKEND.activity_url(obj_id), obj_id) try: @@ -558,7 +574,6 @@ class BaseActivity(object, metaclass=_ActivityMeta): for recp in recipients: logger.debug(f'posting to {recp}') - # ABC BACKEND.post_to_remote_inbox(payload, recp) def _recipients(self) -> List[str]: @@ -665,6 +680,7 @@ class Follow(BaseActivity): def _build_reply(self, reply_type: ActivityType) -> BaseActivity: if reply_type == ActivityType.ACCEPT: return Accept( + actor=self.get_object().id, object=self.to_dict(embed=True), ) @@ -673,7 +689,7 @@ class Follow(BaseActivity): def _recipients(self) -> List[str]: return [self.get_object().id] - def _process_from_inbox(self) -> None: + def _process_from_inbox(self, as_actor: 'Person') -> None: """Receiving a Follow should trigger an Accept.""" accept = self.build_accept() accept.post_to_outbox() @@ -681,10 +697,11 @@ class Follow(BaseActivity): remote_actor = self.get_actor() # ABC - self.new_follower(remote_actor) + BACKEND.new_follower(as_actor, remote_actor) def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - BACKEND.new_following(self.get_actor(), self.get_object()) + # XXX The new_following event will be triggered by Accept + pass def _undo_inbox(self) -> None: # ABC @@ -710,13 +727,12 @@ class Accept(BaseActivity): def _recipients(self) -> List[str]: return [self.get_object().get_actor().id] - def _pre_process_from_inbox(self) -> None: + def _pre_process_from_inbox(self, as_actor: 'Person') -> None: # FIXME(tsileo): ensure the actor match the object actor pass - def _process_from_inbox(self) -> None: - # ABC - self.new_following(self.get_actor().id) + def _process_from_inbox(self, as_actor: 'Person') -> None: + BACKEND.new_following(as_actor, self.get_actor()) class Undo(BaseActivity): @@ -734,14 +750,14 @@ class Undo(BaseActivity): # TODO(tsileo): handle like and announce raise Exception('TODO') - def _pre_process_from_inbox(self) -> None: + def _pre_process_from_inbox(self, as_actor: 'Person') -> None: """Ensures an Undo activity comes from the same actor as the updated activity.""" obj = self.get_object() actor = self.get_actor() if actor.id != obj.get_actor().id: raise BadActivityError(f'{actor!r} cannot update {obj!r}') - def _process_from_inbox(self) -> None: + def _process_from_inbox(self, as_actor: 'Person') -> None: obj = self.get_object() # FIXME(tsileo): move this to _undo_inbox impl # DB.inbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) @@ -783,7 +799,7 @@ class Like(BaseActivity): def _recipients(self) -> List[str]: return [self.get_object().get_actor().id] - def _process_from_inbox(self): + def _process_from_inbox(self, as_actor: 'Person') -> None: # ABC self.inbox_like(self) @@ -821,7 +837,7 @@ class Announce(BaseActivity): return recipients - def _process_from_inbox(self) -> None: + def _process_from_inbox(self, as_actor: 'Person') -> None: if isinstance(self._data['object'], str) and not self._data['object'].startswith('http'): # TODO(tsileo): actually drop it without storing it and better logging, also move the check somewhere else logger.warn( @@ -864,14 +880,14 @@ class Delete(BaseActivity): obj = self._get_actual_object() return obj._recipients() - def _pre_process_from_inbox(self) -> None: + def _pre_process_from_inbox(self, as_actor: 'Person') -> None: """Ensures a Delete activity comes from the same actor as the deleted activity.""" obj = self._get_actual_object() actor = self.get_actor() if actor.id != obj.get_actor().id: raise BadActivityError(f'{actor!r} cannot delete {obj!r}') - def _process_from_inbox(self) -> None: + def _process_from_inbox(self, as_actor: 'Person') -> None: # ABC self.inbox_delete(self) # FIXME(tsileo): handle the delete_threads here? @@ -894,14 +910,14 @@ class Update(BaseActivity): OBJECT_REQUIRED = True ACTOR_REQUIRED = True - def _pre_process_from_inbox(self) -> None: + def _pre_process_from_inbox(self, as_actor: 'Person') -> None: """Ensures an Update activity comes from the same actor as the updated activity.""" obj = self.get_object() actor = self.get_actor() if actor.id != obj.get_actor().id: raise BadActivityError(f'{actor!r} cannot update {obj!r}') - def _process_from_inbox(self): + def _process_from_inbox(self, as_actor: 'Person') -> None: # ABC self.inbox_update(self) @@ -950,7 +966,7 @@ class Create(BaseActivity): return recipients - def _process_from_inbox(self) -> None: + def _process_from_inbox(self, as_actor: 'Person') -> None: # ABC self.inbox_create(self) diff --git a/test_little_boxes.py b/test_little_boxes.py new file mode 100644 index 0000000..b297c8f --- /dev/null +++ b/test_little_boxes.py @@ -0,0 +1,26 @@ +from little_boxes.activitypub import use_backend +from little_boxes.activitypub import BaseBackend +from little_boxes.activitypub import Outbox +from little_boxes.activitypub import Person +from little_boxes.activitypub import Follow + +def test_little_boxes_follow(): + back = BaseBackend() + use_backend(back) + + me = back.setup_actor('Thomas', 'tom') + + other = back.setup_actor('Thomas', 'tom2') + + outbox = Outbox(me) + f = Follow( + actor=me.id, + object=other.id, + ) + + outbox.post(f) + assert back.followers(other) == [me.id] + assert back.following(other) == [] + + assert back.followers(me) == [] + assert back.following(me) == [other.id] From eb25a28679914493a72195e2088a472e51ab2d0f Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 10 Jun 2018 17:08:13 +0200 Subject: [PATCH 0110/1425] Tweak the README --- little_boxes/README.md | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/little_boxes/README.md b/little_boxes/README.md index 021a0f2..024e073 100644 --- a/little_boxes/README.md +++ b/little_boxes/README.md @@ -5,11 +5,7 @@ Tiny ActivityPub framework written in Python, both database and server agnostic. ## Getting Started ```python -from little_boxes.activitypub import BaseBackend -from little_boxes.activitypub import use_backend -from little_boxes.activitypub import Outbox -from little_boxes.activitypub import Person -from little_boxes.activitypub import Follow +from little_boxes import activitypub as ap from mydb import db_client @@ -31,11 +27,11 @@ class MyBackend(BaseBackend): db_con = db_client() my_backend = MyBackend(db_con) -use_backend(my_backend) +ap.use_backend(my_backend) -me = Person({}) # Init an actor -outbox = Outbox(me) +me = ap.Person({}) # Init an actor +outbox = ap.Outbox(me) -follow = Follow(actor=me, object='http://iri-i-want-follow') +follow = ap.Follow(actor=me, object='http://iri-i-want-follow') outbox.post(follow) ``` From c5295524c74c12086f3370047c18ea03debe56a1 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 16 Jun 2018 21:24:53 +0200 Subject: [PATCH 0111/1425] Start migration to little_boxes --- activitypub.py | 1328 +++++++---------------------------- app.py | 174 ++--- config.py | 28 +- little_boxes/README.md | 37 - little_boxes/__init__.py | 12 - little_boxes/activitypub.py | 1073 ---------------------------- little_boxes/errors.py | 55 -- little_boxes/urlutils.py | 47 -- little_boxes/utils.py | 60 -- requirements.txt | 6 +- test_little_boxes.py | 26 - utils/key.py | 51 +- 12 files changed, 370 insertions(+), 2527 deletions(-) delete mode 100644 little_boxes/README.md delete mode 100644 little_boxes/__init__.py delete mode 100644 little_boxes/activitypub.py delete mode 100644 little_boxes/errors.py delete mode 100644 little_boxes/urlutils.py delete mode 100644 little_boxes/utils.py delete mode 100644 test_little_boxes.py diff --git a/activitypub.py b/activitypub.py index cdb0bd1..697ec59 100644 --- a/activitypub.py +++ b/activitypub.py @@ -1,77 +1,29 @@ import logging -import json -import binascii -import os + from datetime import datetime -from enum import Enum from bson.objectid import ObjectId from html2text import html2text from feedgen.feed import FeedGenerator -from utils.actor_service import NotAnActorError -from utils.errors import BadActivityError -from utils.errors import UnexpectedActivityTypeError -from utils.errors import NotFromOutboxError -from utils import activitypub_utils +from little_boxes import activitypub as ap +from little_boxes.backend import Backend +from little_boxes.collection import parse_collection as ap_parse_collection + from config import USERNAME, BASE_URL, ID -from config import CTX_AS, CTX_SECURITY, AS_PUBLIC -from config import DB, ME, ACTOR_SERVICE -from config import OBJECT_SERVICE -from config import PUBLIC_INSTANCES +from config import DB, ME import tasks from typing import List, Optional, Dict, Any, Union logger = logging.getLogger(__name__) -# Helper/shortcut for typing -ObjectType = Dict[str, Any] -ObjectOrIDType = Union[str, ObjectType] - -COLLECTION_CTX = [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - { - "Hashtag": "as:Hashtag", - "sensitive": "as:sensitive", - } -] - - -class ActivityType(Enum): - """Supported activity `type`.""" - ANNOUNCE = 'Announce' - BLOCK = 'Block' - LIKE = 'Like' - CREATE = 'Create' - UPDATE = 'Update' - PERSON = 'Person' - ORDERED_COLLECTION = 'OrderedCollection' - ORDERED_COLLECTION_PAGE = 'OrderedCollectionPage' - COLLECTION_PAGE = 'CollectionPage' - COLLECTION = 'Collection' - NOTE = 'Note' - ACCEPT = 'Accept' - REJECT = 'Reject' - FOLLOW = 'Follow' - DELETE = 'Delete' - UNDO = 'Undo' - IMAGE = 'Image' - TOMBSTONE = 'Tombstone' - - -def random_object_id() -> str: - """Generates a random object ID.""" - return binascii.hexlify(os.urandom(8)).decode('utf-8') - - -def _remove_id(doc: ObjectType) -> ObjectType: +def _remove_id(doc: ap.ObjectType) -> ap.ObjectType: """Helper for removing MongoDB's `_id` field.""" doc = doc.copy() - if '_id' in doc: - del(doc['_id']) + if "_id" in doc: + del (doc["_id"]) return doc @@ -82,1125 +34,337 @@ def _to_list(data: Union[List[Any], Any]) -> List[Any]: return [data] -def clean_activity(activity: ObjectType) -> Dict[str, Any]: - """Clean the activity before rendering it. - - Remove the hidden bco and bcc field - """ - for field in ['bto', 'bcc']: - if field in activity: - del(activity[field]) - if activity['type'] == 'Create' and field in activity['object']: - del(activity['object'][field]) - return activity - - -def _get_actor_id(actor: ObjectOrIDType) -> str: - """Helper for retrieving an actor `id`.""" - if isinstance(actor, dict): - return actor['id'] - return actor - - -class BaseActivity(object): - """Base class for ActivityPub activities.""" - - ACTIVITY_TYPE: Optional[ActivityType] = None - ALLOWED_OBJECT_TYPES: List[ActivityType] = [] - OBJECT_REQUIRED = False - - def __init__(self, **kwargs) -> None: - # Ensure the class has an activity type defined - if not self.ACTIVITY_TYPE: - raise BadActivityError('Missing ACTIVITY_TYPE') - - # XXX(tsileo): what to do about this check? - # Ensure the activity has a type and a valid one - # if kwargs.get('type') is None: - # raise BadActivityError('missing activity type') - - if kwargs.get('type') and kwargs.pop('type') != self.ACTIVITY_TYPE.value: - raise UnexpectedActivityTypeError(f'Expect the type to be {self.ACTIVITY_TYPE.value!r}') - - # Initialize the object - self._data: Dict[str, Any] = { - 'type': self.ACTIVITY_TYPE.value - } - logger.debug(f'initializing a {self.ACTIVITY_TYPE.value} activity: {kwargs}') - - if 'id' in kwargs: - self._data['id'] = kwargs.pop('id') - - if self.ACTIVITY_TYPE != ActivityType.PERSON: - actor = kwargs.get('actor') - if actor: - kwargs.pop('actor') - actor = self._validate_person(actor) - self._data['actor'] = actor - else: - # FIXME(tsileo): uses a special method to set the actor as "the instance" - if not self.NO_CONTEXT and self.ACTIVITY_TYPE != ActivityType.TOMBSTONE: - actor = ID - self._data['actor'] = actor - - if 'object' in kwargs: - obj = kwargs.pop('object') - if isinstance(obj, str): - self._data['object'] = obj - else: - if not self.ALLOWED_OBJECT_TYPES: - raise UnexpectedActivityTypeError('unexpected object') - if 'type' not in obj or (self.ACTIVITY_TYPE != ActivityType.CREATE and 'id' not in obj): - raise BadActivityError('invalid object, missing type') - if ActivityType(obj['type']) not in self.ALLOWED_OBJECT_TYPES: - raise UnexpectedActivityTypeError( - f'unexpected object type {obj["type"]} (allowed={self.ALLOWED_OBJECT_TYPES})' - ) - self._data['object'] = obj - - if '@context' not in kwargs: - if not self.NO_CONTEXT: - self._data['@context'] = CTX_AS - else: - self._data['@context'] = kwargs.pop('@context') - - # @context check - if not self.NO_CONTEXT: - if not isinstance(self._data['@context'], list): - self._data['@context'] = [self._data['@context']] - if CTX_SECURITY not in self._data['@context']: - self._data['@context'].append(CTX_SECURITY) - if isinstance(self._data['@context'][-1], dict): - self._data['@context'][-1]['Hashtag'] = 'as:Hashtag' - self._data['@context'][-1]['sensitive'] = 'as:sensitive' - else: - self._data['@context'].append({'Hashtag': 'as:Hashtag', 'sensitive': 'as:sensitive'}) - - allowed_keys = None - try: - allowed_keys = self._init(**kwargs) - logger.debug('calling custom init') - except NotImplementedError: - pass - - if allowed_keys: - # Allows an extra to (like for Accept and Follow) - kwargs.pop('to', None) - if len(set(kwargs.keys()) - set(allowed_keys)) > 0: - raise BadActivityError(f'extra data left: {kwargs!r}') - else: - # Remove keys with `None` value - valid_kwargs = {} - for k, v in kwargs.items(): - if v is None: - continue - valid_kwargs[k] = v - self._data.update(**valid_kwargs) - - def _init(self, **kwargs) -> Optional[List[str]]: - raise NotImplementedError - - def _verify(self) -> None: - raise NotImplementedError - - def verify(self) -> None: - """Verifies that the activity is valid.""" - if self.OBJECT_REQUIRED and 'object' not in self._data: - raise BadActivityError('activity must have an "object"') - - try: - self._verify() - except NotImplementedError: - pass - - def __repr__(self) -> str: - return '{}({!r})'.format(self.__class__.__qualname__, self._data.get('id')) - - def __str__(self) -> str: - return str(self._data.get('id', f'[new {self.ACTIVITY_TYPE} activity]')) - - def __getattr__(self, name: str) -> Any: - if self._data.get(name): - return self._data.get(name) - - @property - def type_enum(self) -> ActivityType: - return ActivityType(self.type) - - def _set_id(self, uri: str, obj_id: str) -> None: - raise NotImplementedError - - def set_id(self, uri: str, obj_id: str) -> None: - logger.debug(f'setting ID {uri} / {obj_id}') - self._data['id'] = uri - try: - self._set_id(uri, obj_id) - except NotImplementedError: - pass - - def _actor_id(self, obj: ObjectOrIDType) -> str: - if isinstance(obj, dict) and obj['type'] == ActivityType.PERSON.value: - obj_id = obj.get('id') - if not obj_id: - raise ValueError('missing object id') - return obj_id - else: - return str(obj) - - def _validate_person(self, obj: ObjectOrIDType) -> str: - obj_id = self._actor_id(obj) - try: - actor = ACTOR_SERVICE.get(obj_id) - except Exception: - return obj_id # FIXME(tsileo): handle this - if not actor: - raise ValueError('Invalid actor') - return actor['id'] - - def get_object(self) -> 'BaseActivity': - if self.__obj: - return self.__obj - if isinstance(self._data['object'], dict): - p = parse_activity(self._data['object']) - else: - if self.ACTIVITY_TYPE == ActivityType.FOLLOW: - p = Person(**ACTOR_SERVICE.get(self._data['object'])) - else: - obj = OBJECT_SERVICE.get(self._data['object']) - if ActivityType(obj.get('type')) not in self.ALLOWED_OBJECT_TYPES: - raise UnexpectedActivityTypeError(f'invalid object type {obj.get("type")}') - - p = parse_activity(obj) - - self.__obj: Optional[BaseActivity] = p - return p - - def reset_object_cache(self) -> None: - self.__obj = None - - def to_dict(self, embed: bool = False, embed_object_id_only: bool = False) -> ObjectType: - data = dict(self._data) - if embed: - for k in ['@context', 'signature']: - if k in data: - del(data[k]) - if data.get('object') and embed_object_id_only and isinstance(data['object'], dict): - try: - data['object'] = data['object']['id'] - except KeyError: - raise BadActivityError('embedded object does not have an id') - - return data - - def get_actor(self) -> 'BaseActivity': - actor = self._data.get('actor') - if not actor: - if self.type_enum == ActivityType.NOTE: - actor = str(self._data.get('attributedTo')) - else: - raise ValueError('failed to fetch actor') - - actor_id = self._actor_id(actor) - return Person(**ACTOR_SERVICE.get(actor_id)) - - def _pre_post_to_outbox(self) -> None: - raise NotImplementedError - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - raise NotImplementedError - - def _undo_outbox(self) -> None: - raise NotImplementedError - - def _pre_process_from_inbox(self) -> None: - raise NotImplementedError - - def _process_from_inbox(self) -> None: - raise NotImplementedError - - def _undo_inbox(self) -> None: - raise NotImplementedError - - def _undo_should_purge_cache(self) -> bool: - raise NotImplementedError - - def _should_purge_cache(self) -> bool: - raise NotImplementedError - - def process_from_inbox(self) -> None: - logger.debug(f'calling main process from inbox hook for {self}') - self.verify() - actor = self.get_actor() - - # Check for Block activity - if DB.outbox.find_one({'type': ActivityType.BLOCK.value, - 'activity.object': actor.id, - 'meta.undo': False}): - logger.info(f'actor {actor} is blocked, dropping the received activity {self}') - return - - if DB.inbox.find_one({'remote_id': self.id}): - # The activity is already in the inbox - logger.info(f'received duplicate activity {self}, dropping it') - return - - try: - self._pre_process_from_inbox() - logger.debug('called pre process from inbox hook') - except NotImplementedError: - logger.debug('pre process from inbox hook not implemented') - - activity = self.to_dict() - DB.inbox.insert_one({ - 'activity': activity, - 'type': self.type, - 'remote_id': self.id, - 'meta': {'undo': False, 'deleted': False}, - }) - logger.info('activity {self} saved') - - try: - self._process_from_inbox() - logger.debug('called process from inbox hook') - except NotImplementedError: - logger.debug('process from inbox hook not implemented') - - def post_to_outbox(self) -> None: - logger.debug(f'calling main post to outbox hook for {self}') - obj_id = random_object_id() - self.set_id(f'{ID}/outbox/{obj_id}', obj_id) - self.verify() - - try: - self._pre_post_to_outbox() - logger.debug(f'called pre post to outbox hook') - except NotImplementedError: - logger.debug('pre post to outbox hook not implemented') - - activity = self.to_dict() - DB.outbox.insert_one({ - 'id': obj_id, - 'activity': activity, - 'type': self.type, - 'remote_id': self.id, - 'meta': {'undo': False, 'deleted': False}, - }) - - recipients = self.recipients() - logger.info(f'recipients={recipients}') - activity = clean_activity(activity) - - try: - self._post_to_outbox(obj_id, activity, recipients) - logger.debug(f'called post to outbox hook') - except NotImplementedError: - logger.debug('post to outbox hook not implemented') - - payload = json.dumps(activity) - for recp in recipients: - logger.debug(f'posting to {recp}') - self._post_to_inbox(payload, recp) - - def _post_to_inbox(self, payload: str, to: str): - tasks.post_to_inbox.delay(payload, to) - - def _recipients(self) -> List[str]: - return [] - - def recipients(self) -> List[str]: - recipients = self._recipients() - - out: List[str] = [] - for recipient in recipients: - if recipient in PUBLIC_INSTANCES: - if recipient not in out: - out.append(str(recipient)) - continue - if recipient in [ME, AS_PUBLIC, None]: - continue - if isinstance(recipient, Person): - if recipient.id == ME: - continue - actor = recipient - else: - try: - actor = Person(**ACTOR_SERVICE.get(recipient)) - - if actor.endpoints: - shared_inbox = actor.endpoints.get('sharedInbox') - if shared_inbox not in out: - out.append(shared_inbox) - continue - - if actor.inbox and actor.inbox not in out: - out.append(actor.inbox) - - except NotAnActorError as error: - # Is the activity a `Collection`/`OrderedCollection`? - if error.activity and error.activity['type'] in [ActivityType.COLLECTION.value, - ActivityType.ORDERED_COLLECTION.value]: - for item in parse_collection(error.activity): - if item in [ME, AS_PUBLIC]: - continue - try: - col_actor = Person(**ACTOR_SERVICE.get(item)) - except NotAnActorError: - pass - - if col_actor.endpoints: - shared_inbox = col_actor.endpoints.get('sharedInbox') - if shared_inbox not in out: - out.append(shared_inbox) - continue - if col_actor.inbox and col_actor.inbox not in out: - out.append(col_actor.inbox) - - return out - - def build_undo(self) -> 'BaseActivity': - raise NotImplementedError - - def build_delete(self) -> 'BaseActivity': - raise NotImplementedError - - -class Person(BaseActivity): - ACTIVITY_TYPE = ActivityType.PERSON - - def _init(self, **kwargs): - # if 'icon' in kwargs: - # self._data['icon'] = Image(**kwargs.pop('icon')) - pass - - def _verify(self) -> None: - ACTOR_SERVICE.get(self._data['id']) - - -class Block(BaseActivity): - ACTIVITY_TYPE = ActivityType.BLOCK - OBJECT_REQUIRED = True - - -class Collection(BaseActivity): - ACTIVITY_TYPE = ActivityType.COLLECTION - - -class Image(BaseActivity): - ACTIVITY_TYPE = ActivityType.IMAGE - NO_CONTEXT = True - - def _init(self, **kwargs): - self._data.update( - url=kwargs.pop('url'), +class MicroblogPubBackend(Backend): + def base_url(self) -> str: + return BASE_URL + + def activity_url(self, obj_id): + return f"{BASE_URL}/outbox/{obj_id}" + + def outbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: + DB.outbox.insert_one( + { + "activity": activity.to_dict(), + "type": activity.type, + "remote_id": activity.id, + "meta": {"undo": False, "deleted": False}, + } ) - def __repr__(self): - return 'Image({!r})'.format(self._data.get('url')) - - -class Follow(BaseActivity): - ACTIVITY_TYPE = ActivityType.FOLLOW - ALLOWED_OBJECT_TYPES = [ActivityType.PERSON] - OBJECT_REQUIRED = True - - def _build_reply(self, reply_type: ActivityType) -> BaseActivity: - if reply_type == ActivityType.ACCEPT: - return Accept( - object=self.to_dict(embed=True), + def outbox_is_blocked(self, as_actor: ap.Person, actor_id: str) -> bool: + return bool( + DB.outbox.find_one( + { + "type": ap.ActivityType.BLOCK.value, + "activity.object": as_actor.id, + "meta.undo": False, + } ) + ) - raise ValueError(f'type {reply_type} is invalid for building a reply') + def fetch_iri(self, iri: str) -> ap.ObjectType: + pass - def _recipients(self) -> List[str]: - return [self.get_object().id] + def inbox_check_duplicate(self, as_actor: ap.Person, iri: str) -> bool: + return bool(DB.inbox.find_one({"remote_id": iri})) - def _process_from_inbox(self) -> None: - accept = self.build_accept() - accept.post_to_outbox() + def inbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: + DB.inbox.insert_one( + { + "activity": activity.to_dict(), + "type": activity.type, + "remote_id": activity.id, + "meta": {"undo": False, "deleted": False}, + } + ) - remote_actor = self.get_actor().id + def post_to_remote_inbox(self, as_actor: ap.Person, payload: str, to: str) -> None: + tasks.post_to_inbox.delay(payload, to) - if DB.followers.find({'remote_actor': remote_actor}).count() == 0: - DB.followers.insert_one({'remote_actor': remote_actor}) + def new_follower(self, as_actor: ap.Person, follow: ap.Follow) -> None: + remote_actor = follow.get_actor().id - def _undo_inbox(self) -> None: - DB.followers.delete_one({'remote_actor': self.get_actor().id}) + if DB.followers.find({"remote_actor": remote_actor}).count() == 0: + DB.followers.insert_one({"remote_actor": remote_actor}) - def _undo_outbox(self) -> None: - DB.following.delete_one({'remote_actor': self.get_object().id}) + def undo_new_follower(self, as_actor: ap.Person, follow: ap.Follow) -> None: + # TODO(tsileo): update the follow to set undo + DB.followers.delete_one({"remote_actor": follow.get_actor().id}) - def build_accept(self) -> BaseActivity: - return self._build_reply(ActivityType.ACCEPT) + def undo_new_following(self, as_actor: ap.Person, follow: ap.Follow) -> None: + # TODO(tsileo): update the follow to set undo + DB.following.delete_one({"remote_actor": follow.get_object().id}) - def build_undo(self) -> BaseActivity: - return Undo(object=self.to_dict(embed=True)) + def new_following(self, as_actor: ap.Person, follow: ap.Follow) -> None: + remote_actor = follow.get_actor().id + if DB.following.find({"remote_actor": remote_actor}).count() == 0: + DB.following.insert_one({"remote_actor": remote_actor}) - def _should_purge_cache(self) -> bool: - # Receiving a follow activity in the inbox should reset the application cache - return True - - -class Accept(BaseActivity): - ACTIVITY_TYPE = ActivityType.ACCEPT - ALLOWED_OBJECT_TYPES = [ActivityType.FOLLOW] - - def _recipients(self) -> List[str]: - return [self.get_object().get_actor().id] - - def _process_from_inbox(self) -> None: - remote_actor = self.get_actor().id - if DB.following.find({'remote_actor': remote_actor}).count() == 0: - DB.following.insert_one({'remote_actor': remote_actor}) - - def _should_purge_cache(self) -> bool: - # Receiving an accept activity in the inbox should reset the application cache - # (a follow request has been accepted) - return True - - -class Undo(BaseActivity): - ACTIVITY_TYPE = ActivityType.UNDO - ALLOWED_OBJECT_TYPES = [ActivityType.FOLLOW, ActivityType.LIKE, ActivityType.ANNOUNCE] - OBJECT_REQUIRED = True - - def _recipients(self) -> List[str]: - obj = self.get_object() - if obj.type_enum == ActivityType.FOLLOW: - return [obj.get_object().id] - else: - return [obj.get_object().get_actor().id] - # TODO(tsileo): handle like and announce - raise Exception('TODO') - - def _pre_process_from_inbox(self) -> None: - """Ensures an Undo activity comes from the same actor as the updated activity.""" - obj = self.get_object() - actor = self.get_actor() - if actor.id != obj.get_actor().id: - raise BadActivityError(f'{actor!r} cannot update {obj!r}') - - def _process_from_inbox(self) -> None: - obj = self.get_object() - DB.inbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) - - try: - obj._undo_inbox() - except NotImplementedError: - pass - - def _should_purge_cache(self) -> bool: - obj = self.get_object() - try: - # Receiving a undo activity regarding an activity that was mentioning a published activity - # should purge the cache - return obj._undo_should_purge_cache() - except NotImplementedError: - pass - - return False - - def _pre_post_to_outbox(self) -> None: - """Ensures an Undo activity references an activity owned by the instance.""" - obj = self.get_object() - if not obj.id.startswith(ID): - raise NotFromOutboxError(f'object {obj["id"]} is not owned by this instance') - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - logger.debug('processing undo to outbox') - logger.debug('self={}'.format(self)) - obj = self.get_object() - logger.debug('obj={}'.format(obj)) - DB.outbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) - - try: - obj._undo_outbox() - logger.debug(f'_undo_outbox called for {obj}') - except NotImplementedError: - logger.debug(f'_undo_outbox not implemented for {obj}') - pass - - -class Like(BaseActivity): - ACTIVITY_TYPE = ActivityType.LIKE - ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] - OBJECT_REQUIRED = True - - def _recipients(self) -> List[str]: - return [self.get_object().get_actor().id] - - def _process_from_inbox(self): - obj = self.get_object() + def inbox_like(self, as_actor: ap.Person, like: ap.Like) -> None: + obj = like.get_object() # Update the meta counter if the object is published by the server - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_like': 1}, - }) - # XXX(tsileo): notification?? + DB.outbox.update_one( + {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": 1}} + ) - def _undo_inbox(self) -> None: - obj = self.get_object() + def inbox_undo_like(self, as_actor: ap.Person, like: ap.Like) -> None: + obj = like.get_object() # Update the meta counter if the object is published by the server - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_like': -1}, - }) + DB.outbox.update_one( + {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": -1}} + ) - def _undo_should_purge_cache(self) -> bool: - # If a like coutn was decremented, we need to purge the application cache - return self.get_object().id.startswith(BASE_URL) - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]): - obj = self.get_object() + def outobx_like(self, as_actor: ap.Person, like: ap.Like) -> None: + obj = like.get_object() # Unlikely, but an actor can like it's own post - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_like': 1}, - }) + DB.outbox.update_one( + {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": 1}} + ) # Keep track of the like we just performed - DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.liked': obj_id}}) + DB.inbox.update_one( + {"activity.object.id": obj.id}, {"$set": {"meta.liked": like.id}} + ) - def _undo_outbox(self) -> None: - obj = self.get_object() + def outbox_undo_like(self, as_actor: ap.Person, like: ap.Like) -> None: + obj = like.get_object() # Unlikely, but an actor can like it's own post - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_like': -1}, - }) + DB.outbox.update_one( + {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": -1}} + ) - DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.liked': False}}) + DB.inbox.update_one( + {"activity.object.id": obj.id}, {"$set": {"meta.liked": False}} + ) - def build_undo(self) -> BaseActivity: - return Undo(object=self.to_dict(embed=True, embed_object_id_only=True)) - - -class Announce(BaseActivity): - ACTIVITY_TYPE = ActivityType.ANNOUNCE - ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] - - def _recipients(self) -> List[str]: - recipients = [] - - for field in ['to', 'cc']: - if field in self._data: - recipients.extend(_to_list(self._data[field])) - - return recipients - - def _process_from_inbox(self) -> None: - if isinstance(self._data['object'], str) and not self._data['object'].startswith('http'): + def inbox_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: + if isinstance(announce._data["object"], str) and not announce._data[ + "object" + ].startswith("http"): # TODO(tsileo): actually drop it without storing it and better logging, also move the check somewhere else logger.warn( - f'received an Annouce referencing an OStatus notice ({self._data["object"]}), dropping the message' + f'received an Annouce referencing an OStatus notice ({announce._data["object"]}), dropping the message' ) return - # Save/cache the object, and make it part of the stream so we can fetch it - if isinstance(self._data['object'], str): - raw_obj = OBJECT_SERVICE.get( - self._data['object'], - reload_cache=True, - part_of_stream=True, - announce_published=self._data['published'], - ) - obj = parse_activity(raw_obj) + # FIXME(tsileo): Save/cache the object, and make it part of the stream so we can fetch it + if isinstance(announce._data["object"], str): + obj_iri = announce._data["object"] else: - obj = self.get_object() + obj_iri = self.get_object().id - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_boost': 1}, - }) + DB.outbox.update_one( + {"activity.object.id": obj_iri}, {"$inc": {"meta.count_boost": 1}} + ) - def _undo_inbox(self) -> None: - obj = self.get_object() + def inbox_undo_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: + obj = announce.get_object() # Update the meta counter if the object is published by the server - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_boost': -1}, - }) + DB.outbox.update_one( + {"activity.object.id": obj.id}, {"$inc": {"meta.count_boost": -1}} + ) - def _undo_should_purge_cache(self) -> bool: - # If a like coutn was decremented, we need to purge the application cache - return self.get_object().id.startswith(BASE_URL) + def outbox_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: + obj = announce.get_object() + DB.inbox.update_one( + {"activity.object.id": obj.id}, {"$set": {"meta.boosted": announce.id}} + ) - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - if isinstance(self._data['object'], str): - # Put the object in the cache - OBJECT_SERVICE.get( - self._data['object'], - reload_cache=True, - part_of_stream=True, - announce_published=self._data['published'], - ) + def outbox_undo_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: + obj = announce.get_object() + DB.inbox.update_one( + {"activity.object.id": obj.id}, {"$set": {"meta.boosted": False}} + ) - obj = self.get_object() - DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.boosted': obj_id}}) - - def _undo_outbox(self) -> None: - obj = self.get_object() - DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.boosted': False}}) - - def build_undo(self) -> BaseActivity: - return Undo(object=self.to_dict(embed=True)) - - -class Delete(BaseActivity): - ACTIVITY_TYPE = ActivityType.DELETE - ALLOWED_OBJECT_TYPES = [ActivityType.NOTE, ActivityType.TOMBSTONE] - OBJECT_REQUIRED = True - - def _get_actual_object(self) -> BaseActivity: - obj = self.get_object() - if obj.type_enum == ActivityType.TOMBSTONE: - obj = parse_activity(OBJECT_SERVICE.get(obj.id)) - return obj - - def _recipients(self) -> List[str]: - obj = self._get_actual_object() - return obj._recipients() - - def _pre_process_from_inbox(self) -> None: - """Ensures a Delete activity comes from the same actor as the deleted activity.""" - obj = self._get_actual_object() - actor = self.get_actor() - if actor.id != obj.get_actor().id: - raise BadActivityError(f'{actor!r} cannot delete {obj!r}') - - def _process_from_inbox(self) -> None: - DB.inbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) - obj = self._get_actual_object() - if obj.type_enum == ActivityType.NOTE: - obj._delete_from_threads() + def inbox_delete(self, as_actor: ap.Person, delete: ap.Delete) -> None: + DB.inbox.update_one( + {"activity.object.id": delete.get_object().id}, + {"$set": {"meta.deleted": True}}, + ) + # FIXME(tsileo): handle threads + # obj = delete._get_actual_object() + # if obj.type_enum == ActivityType.NOTE: + # obj._delete_from_threads() # TODO(tsileo): also purge the cache if it's a reply of a published activity - def _pre_post_to_outbox(self) -> None: - """Ensures the Delete activity references a activity from the outbox (i.e. owned by the instance).""" - obj = self._get_actual_object() - if not obj.id.startswith(ID): - raise NotFromOutboxError(f'object {obj["id"]} is not owned by this instance') + def outbox_delete(self, as_actor: ap.Person, delete: ap.Delete) -> None: + DB.outbox.update_one( + {"activity.object.id": delete.get_object().id}, + {"$set": {"meta.deleted": True}}, + ) - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - DB.outbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) - - -class Update(BaseActivity): - ACTIVITY_TYPE = ActivityType.UPDATE - ALLOWED_OBJECT_TYPES = [ActivityType.NOTE, ActivityType.PERSON] - OBJECT_REQUIRED = True - - def _pre_process_from_inbox(self) -> None: - """Ensures an Update activity comes from the same actor as the updated activity.""" - obj = self.get_object() - actor = self.get_actor() - if actor.id != obj.get_actor().id: - raise BadActivityError(f'{actor!r} cannot update {obj!r}') - - def _process_from_inbox(self): - obj = self.get_object() - if obj.type_enum == ActivityType.NOTE: - DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'activity.object': obj.to_dict()}}) + def inbox_update(self, as_actor: ap.Person, update: ap.Update) -> None: + obj = update.get_object() + if obj.ACTIVITY_TYPE == ap.ActivityType.NOTE: + DB.inbox.update_one( + {"activity.object.id": obj.id}, + {"$set": {"activity.object": obj.to_dict()}}, + ) return - # If the object is a Person, it means the profile was updated, we just refresh our local cache - ACTOR_SERVICE.get(obj.id, reload_cache=True) + # FIXME(tsileo): handle update actor amd inbox_update_note/inbox_update_actor - # TODO(tsileo): implements _should_purge_cache if it's a reply of a published activity (i.e. in the outbox) + def outbox_update(self, as_actor: ap.Person, update: ap.Update) -> None: + obj = update._data["object"] - def _pre_post_to_outbox(self) -> None: - obj = self.get_object() - if not obj.id.startswith(ID): - raise NotFromOutboxError(f'object {obj["id"]} is not owned by this instance') - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - obj = self._data['object'] - - update_prefix = 'activity.object.' - update: Dict[str, Any] = {'$set': dict(), '$unset': dict()} - update['$set'][f'{update_prefix}updated'] = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' + update_prefix = "activity.object." + update: Dict[str, Any] = {"$set": dict(), "$unset": dict()} + update["$set"][f"{update_prefix}updated"] = ( + datetime.utcnow().replace(microsecond=0).isoformat() + "Z" + ) for k, v in obj.items(): - if k in ['id', 'type']: + if k in ["id", "type"]: continue if v is None: - update['$unset'][f'{update_prefix}{k}'] = '' + update["$unset"][f"{update_prefix}{k}"] = "" else: - update['$set'][f'{update_prefix}{k}'] = v + update["$set"][f"{update_prefix}{k}"] = v - if len(update['$unset']) == 0: - del(update['$unset']) + if len(update["$unset"]) == 0: + del (update["$unset"]) - print(f'updating note from outbox {obj!r} {update}') - logger.info(f'updating note from outbox {obj!r} {update}') - DB.outbox.update_one({'activity.object.id': obj['id']}, update) + print(f"updating note from outbox {obj!r} {update}") + logger.info(f"updating note from outbox {obj!r} {update}") + DB.outbox.update_one({"activity.object.id": obj["id"]}, update) # FIXME(tsileo): should send an Update (but not a partial one, to all the note's recipients # (create a new Update with the result of the update, and send it without saving it?) -class Create(BaseActivity): - ACTIVITY_TYPE = ActivityType.CREATE - ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] - OBJECT_REQUIRED = True - - def _set_id(self, uri: str, obj_id: str) -> None: - self._data['object']['id'] = uri + '/activity' - self._data['object']['url'] = ID + '/' + self.get_object().type.lower() + '/' + obj_id - self.reset_object_cache() - - def _init(self, **kwargs): - obj = self.get_object() - if not obj.attributedTo: - self._data['object']['attributedTo'] = self.get_actor().id - if not obj.published: - if self.published: - self._data['object']['published'] = self.published - else: - now = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' - self._data['published'] = now - self._data['object']['published'] = now - - def _recipients(self) -> List[str]: - # TODO(tsileo): audience support? - recipients = [] - for field in ['to', 'cc', 'bto', 'bcc']: - if field in self._data: - recipients.extend(_to_list(self._data[field])) - - recipients.extend(self.get_object()._recipients()) - - return recipients - - def _update_threads(self) -> None: - logger.debug('_update_threads hook') - obj = self.get_object() - - # TODO(tsileo): re-enable me - # tasks.fetch_og.delay('INBOX', self.id) - - threads = [] - reply = obj.get_local_reply() - print(f'initial_reply={reply}') - print(f'{obj}') - logger.debug(f'initial_reply={reply}') - reply_id = None - direct_reply = 1 - while reply is not None: - if not DB.inbox.find_one_and_update({'activity.object.id': reply.id}, { - '$inc': { - 'meta.count_reply': 1, - 'meta.count_direct_reply': direct_reply, - }, - '$addToSet': {'meta.thread_children': obj.id}, - }): - DB.outbox.update_one({'activity.object.id': reply.id}, { - '$inc': { - 'meta.count_reply': 1, - 'meta.count_direct_reply': direct_reply, - }, - '$addToSet': {'meta.thread_children': obj.id}, - }) - - direct_reply = 0 - reply_id = reply.id - reply = reply.get_local_reply() - logger.debug(f'next_reply={reply}') - threads.append(reply_id) - # FIXME(tsileo): obj.id is None!! - print(f'reply_id={reply_id} {obj.id} {obj._data} {self.id}') - - if reply_id: - if not DB.inbox.find_one_and_update({'activity.object.id': obj.id}, { - '$set': { - 'meta.thread_parents': threads, - 'meta.thread_root_parent': reply_id, - }, - }): - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$set': { - 'meta.thread_parents': threads, - 'meta.thread_root_parent': reply_id, - }, - }) - logger.debug('_update_threads done') - - def _process_from_inbox(self) -> None: - self._update_threads() - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - self._update_threads() - - def _should_purge_cache(self) -> bool: - # TODO(tsileo): handle reply of a reply... - obj = self.get_object() - in_reply_to = obj.inReplyTo - if in_reply_to: - local_activity = DB.outbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) - if local_activity: - return True - - return False - - -class Tombstone(BaseActivity): - ACTIVITY_TYPE = ActivityType.TOMBSTONE - - -class Note(BaseActivity): - ACTIVITY_TYPE = ActivityType.NOTE - - def _init(self, **kwargs): - print(self._data) - # Remove the `actor` field as `attributedTo` is used for `Note` instead - if 'actor' in self._data: - del(self._data['actor']) - if 'sensitive' not in kwargs: - self._data['sensitive'] = False - - def _recipients(self) -> List[str]: - # TODO(tsileo): audience support? - recipients: List[str] = [] - - # If the note is public, we publish it to the defined "public instances" - if AS_PUBLIC in self._data.get('to', []): - recipients.extend(PUBLIC_INSTANCES) - print('publishing to public instances') - print(recipients) - - for field in ['to', 'cc', 'bto', 'bcc']: - if field in self._data: - recipients.extend(_to_list(self._data[field])) - - return recipients - - def _delete_from_threads(self) -> None: - logger.debug('_delete_from_threads hook') - - reply = self.get_local_reply() - logger.debug(f'initial_reply={reply}') - direct_reply = -1 - while reply is not None: - if not DB.inbox.find_one_and_update({'activity.object.id': reply.id}, { - '$inc': { - 'meta.count_reply': -1, - 'meta.count_direct_reply': direct_reply, - }, - '$pull': {'meta.thread_children': self.id}, - - }): - DB.outbox.update_one({'activity.object.id': reply.id}, { - '$inc': { - 'meta.count_reply': 1, - 'meta.count_direct_reply': direct_reply, - }, - '$pull': {'meta.thread_children': self.id}, - }) - - direct_reply = 0 - reply = reply.get_local_reply() - logger.debug(f'next_reply={reply}') - - logger.debug('_delete_from_threads done') - return None - - def get_local_reply(self) -> Optional[BaseActivity]: - "Find the note reply if any.""" - in_reply_to = self.inReplyTo - if not in_reply_to: - # This is the root comment - return None - - inbox_parent = DB.inbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) - if inbox_parent: - return parse_activity(inbox_parent['activity']['object']) - - outbox_parent = DB.outbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) - if outbox_parent: - return parse_activity(outbox_parent['activity']['object']) - - # The parent is no stored on this instance - return None - - def build_create(self) -> BaseActivity: - """Wraps an activity in a Create activity.""" - create_payload = { - 'object': self.to_dict(embed=True), - 'actor': self.attributedTo or ME, - } - for field in ['published', 'to', 'bto', 'cc', 'bcc', 'audience']: - if field in self._data: - create_payload[field] = self._data[field] - - return Create(**create_payload) - - def build_like(self) -> BaseActivity: - return Like(object=self.id) - - def build_announce(self) -> BaseActivity: - return Announce( - object=self.id, - to=[AS_PUBLIC], - cc=[ID+'/followers', self.attributedTo], - published=datetime.utcnow().replace(microsecond=0).isoformat() + 'Z', - ) - - def build_delete(self) -> BaseActivity: - return Delete(object=Tombstone(id=self.id).to_dict(embed=True)) - - def get_tombstone(self, deleted: Optional[str]) -> BaseActivity: - return Tombstone( - id=self.id, - published=self.published, - deleted=deleted, - updated=deleted, - ) - - -_ACTIVITY_TYPE_TO_CLS = { - ActivityType.IMAGE: Image, - ActivityType.PERSON: Person, - ActivityType.FOLLOW: Follow, - ActivityType.ACCEPT: Accept, - ActivityType.UNDO: Undo, - ActivityType.LIKE: Like, - ActivityType.ANNOUNCE: Announce, - ActivityType.UPDATE: Update, - ActivityType.DELETE: Delete, - ActivityType.CREATE: Create, - ActivityType.NOTE: Note, - ActivityType.BLOCK: Block, - ActivityType.COLLECTION: Collection, - ActivityType.TOMBSTONE: Tombstone, -} - - -def parse_activity(payload: ObjectType, expected: Optional[ActivityType] = None) -> BaseActivity: - t = ActivityType(payload['type']) - - if expected and t != expected: - raise UnexpectedActivityTypeError(f'expected a {expected.name} activity, got a {payload["type"]}') - - if t not in _ACTIVITY_TYPE_TO_CLS: - raise BadActivityError(f'unsupported activity type {payload["type"]}') - - activity = _ACTIVITY_TYPE_TO_CLS[t](**payload) - - return activity - - def gen_feed(): fg = FeedGenerator() - fg.id(f'{ID}') - fg.title(f'{USERNAME} notes') - fg.author({'name': USERNAME, 'email': 't@a4.io'}) - fg.link(href=ID, rel='alternate') - fg.description(f'{USERNAME} notes') - fg.logo(ME.get('icon', {}).get('url')) - fg.language('en') - for item in DB.outbox.find({'type': 'Create'}, limit=50): + fg.id(f"{ID}") + fg.title(f"{USERNAME} notes") + fg.author({"name": USERNAME, "email": "t@a4.io"}) + fg.link(href=ID, rel="alternate") + fg.description(f"{USERNAME} notes") + fg.logo(ME.get("icon", {}).get("url")) + fg.language("en") + for item in DB.outbox.find({"type": "Create"}, limit=50): fe = fg.add_entry() - fe.id(item['activity']['object'].get('url')) - fe.link(href=item['activity']['object'].get('url')) - fe.title(item['activity']['object']['content']) - fe.description(item['activity']['object']['content']) + fe.id(item["activity"]["object"].get("url")) + fe.link(href=item["activity"]["object"].get("url")) + fe.title(item["activity"]["object"]["content"]) + fe.description(item["activity"]["object"]["content"]) return fg def json_feed(path: str) -> Dict[str, Any]: """JSON Feed (https://jsonfeed.org/) document.""" data = [] - for item in DB.outbox.find({'type': 'Create'}, limit=50): - data.append({ - "id": item["id"], - "url": item['activity']['object'].get('url'), - "content_html": item['activity']['object']['content'], - "content_text": html2text(item['activity']['object']['content']), - "date_published": item['activity']['object'].get('published'), - }) + for item in DB.outbox.find({"type": "Create"}, limit=50): + data.append( + { + "id": item["id"], + "url": item["activity"]["object"].get("url"), + "content_html": item["activity"]["object"]["content"], + "content_text": html2text(item["activity"]["object"]["content"]), + "date_published": item["activity"]["object"].get("published"), + } + ) return { "version": "https://jsonfeed.org/version/1", - "user_comment": ("This is a microblog feed. You can add this to your feed reader using the following URL: " - + ID + path), + "user_comment": ( + "This is a microblog feed. You can add this to your feed reader using the following URL: " + + ID + + path + ), "title": USERNAME, "home_page_url": ID, "feed_url": ID + path, "author": { "name": USERNAME, "url": ID, - "avatar": ME.get('icon', {}).get('url'), + "avatar": ME.get("icon", {}).get("url"), }, "items": data, } -def build_inbox_json_feed(path: str, request_cursor: Optional[str] = None) -> Dict[str, Any]: +def build_inbox_json_feed( + path: str, request_cursor: Optional[str] = None +) -> Dict[str, Any]: data = [] cursor = None - q: Dict[str, Any] = {'type': 'Create', 'meta.deleted': False} + q: Dict[str, Any] = {"type": "Create", "meta.deleted": False} if request_cursor: - q['_id'] = {'$lt': request_cursor} + q["_id"] = {"$lt": request_cursor} - for item in DB.inbox.find(q, limit=50).sort('_id', -1): - actor = ACTOR_SERVICE.get(item['activity']['actor']) - data.append({ - "id": item["activity"]["id"], - "url": item['activity']['object'].get('url'), - "content_html": item['activity']['object']['content'], - "content_text": html2text(item['activity']['object']['content']), - "date_published": item['activity']['object'].get('published'), - "author": { - "name": actor.get('name', actor.get('preferredUsername')), - "url": actor.get('url'), - 'avatar': actor.get('icon', {}).get('url'), - }, - }) - cursor = str(item['_id']) + for item in DB.inbox.find(q, limit=50).sort("_id", -1): + actor = ap.get_backend().fetch_iri(item["activity"]["actor"]) + data.append( + { + "id": item["activity"]["id"], + "url": item["activity"]["object"].get("url"), + "content_html": item["activity"]["object"]["content"], + "content_text": html2text(item["activity"]["object"]["content"]), + "date_published": item["activity"]["object"].get("published"), + "author": { + "name": actor.get("name", actor.get("preferredUsername")), + "url": actor.get("url"), + "avatar": actor.get("icon", {}).get("url"), + }, + } + ) + cursor = str(item["_id"]) resp = { "version": "https://jsonfeed.org/version/1", - "title": f'{USERNAME}\'s stream', + "title": f"{USERNAME}'s stream", "home_page_url": ID, "feed_url": ID + path, "items": data, } if cursor and len(data) == 50: - resp['next_url'] = ID + path + '?cursor=' + cursor + resp["next_url"] = ID + path + "?cursor=" + cursor return resp -def parse_collection(payload: Optional[Dict[str, Any]] = None, url: Optional[str] = None) -> List[str]: +def parse_collection( + payload: Optional[Dict[str, Any]] = None, url: Optional[str] = None +) -> List[str]: """Resolve/fetch a `Collection`/`OrderedCollection`.""" # Resolve internal collections via MongoDB directly - if url == ID + '/followers': - return [doc['remote_actor'] for doc in DB.followers.find()] - elif url == ID + '/following': - return [doc['remote_actor'] for doc in DB.following.find()] + if url == ID + "/followers": + return [doc["remote_actor"] for doc in DB.followers.find()] + elif url == ID + "/following": + return [doc["remote_actor"] for doc in DB.following.find()] # Go through all the pages - return activitypub_utils.parse_collection(payload, url) + return ap_parse_collection(payload, url) def embed_collection(total_items, first_page_id): return { - "type": ActivityType.ORDERED_COLLECTION.value, + "type": ap.ActivityType.ORDERED_COLLECTION.value, "totalItems": total_items, - "first": f'{first_page_id}?page=first', + "first": f"{first_page_id}?page=first", "id": first_page_id, } -def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, col_name=None, first_page=False): +def build_ordered_collection( + col, q=None, cursor=None, map_func=None, limit=50, col_name=None, first_page=False +): col_name = col_name or col.name if q is None: q = {} if cursor: - q['_id'] = {'$lt': ObjectId(cursor)} - data = list(col.find(q, limit=limit).sort('_id', -1)) + q["_id"] = {"$lt": ObjectId(cursor)} + data = list(col.find(q, limit=limit).sort("_id", -1)) if not data: return { - 'id': BASE_URL + '/' + col_name, - 'totalItems': 0, - 'type': ActivityType.ORDERED_COLLECTION.value, - 'orederedItems': [], + "id": BASE_URL + "/" + col_name, + "totalItems": 0, + "type": ap.ActivityType.ORDERED_COLLECTION.value, + "orederedItems": [], } - start_cursor = str(data[0]['_id']) - next_page_cursor = str(data[-1]['_id']) + start_cursor = str(data[0]["_id"]) + next_page_cursor = str(data[-1]["_id"]) total_items = col.find(q).count() data = [_remove_id(doc) for doc in data] @@ -1210,41 +374,43 @@ def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, # No cursor, this is the first page and we return an OrderedCollection if not cursor: resp = { - '@context': COLLECTION_CTX, - 'id': f'{BASE_URL}/{col_name}', - 'totalItems': total_items, - 'type': ActivityType.ORDERED_COLLECTION.value, - 'first': { - 'id': f'{BASE_URL}/{col_name}?cursor={start_cursor}', - 'orderedItems': data, - 'partOf': f'{BASE_URL}/{col_name}', - 'totalItems': total_items, - 'type': ActivityType.ORDERED_COLLECTION_PAGE.value, + "@context": ap.COLLECTION_CTX, + "id": f"{BASE_URL}/{col_name}", + "totalItems": total_items, + "type": ap.ActivityType.ORDERED_COLLECTION.value, + "first": { + "id": f"{BASE_URL}/{col_name}?cursor={start_cursor}", + "orderedItems": data, + "partOf": f"{BASE_URL}/{col_name}", + "totalItems": total_items, + "type": ap.ActivityType.ORDERED_COLLECTION_PAGE.value, }, } if len(data) == limit: - resp['first']['next'] = BASE_URL + '/' + col_name + '?cursor=' + next_page_cursor + resp["first"]["next"] = ( + BASE_URL + "/" + col_name + "?cursor=" + next_page_cursor + ) if first_page: - return resp['first'] + return resp["first"] return resp # If there's a cursor, then we return an OrderedCollectionPage resp = { - '@context': COLLECTION_CTX, - 'type': ActivityType.ORDERED_COLLECTION_PAGE.value, - 'id': BASE_URL + '/' + col_name + '?cursor=' + start_cursor, - 'totalItems': total_items, - 'partOf': BASE_URL + '/' + col_name, - 'orderedItems': data, + "@context": ap.COLLECTION_CTX, + "type": ap.ActivityType.ORDERED_COLLECTION_PAGE.value, + "id": BASE_URL + "/" + col_name + "?cursor=" + start_cursor, + "totalItems": total_items, + "partOf": BASE_URL + "/" + col_name, + "orderedItems": data, } if len(data) == limit: - resp['next'] = BASE_URL + '/' + col_name + '?cursor=' + next_page_cursor + resp["next"] = BASE_URL + "/" + col_name + "?cursor=" + next_page_cursor if first_page: - return resp['first'] + return resp["first"] # XXX(tsileo): implements prev with prev=? diff --git a/app.py b/app.py index e3cd805..6b104d2 100644 --- a/app.py +++ b/app.py @@ -67,7 +67,7 @@ from utils.errors import ActivityNotFoundError from typing import Dict, Any - + app = Flask(__name__) app.secret_key = get_secret_key('flask') app.config.update( @@ -137,23 +137,23 @@ def clean_html(html): return bleach.clean(html, tags=ALLOWED_TAGS) -@app.template_filter() -def quote_plus(t): - return urllib.parse.quote_plus(t) +@app.template_filter() +def quote_plus(t): + return urllib.parse.quote_plus(t) -@app.template_filter() -def is_from_outbox(t): +@app.template_filter() +def is_from_outbox(t): return t.startswith(ID) -@app.template_filter() -def clean(html): - return clean_html(html) +@app.template_filter() +def clean(html): + return clean_html(html) -@app.template_filter() -def html2plaintext(body): +@app.template_filter() +def html2plaintext(body): return html2text(body) @@ -183,7 +183,7 @@ def format_timeago(val): return timeago.format(datetime.strptime(val, '%Y-%m-%dT%H:%M:%SZ'), datetime.utcnow()) except: return timeago.format(datetime.strptime(val, '%Y-%m-%dT%H:%M:%S.%fZ'), datetime.utcnow()) - + return val def _is_img(filename): @@ -279,7 +279,7 @@ def handle_activitypub_error(error): return response -# App routes +# App routes ####### # Login @@ -487,7 +487,7 @@ def _build_thread(data, include_children=True): def _flatten(node, level=0): node['_level'] = level thread.append(node) - + for snode in sorted(idx[node['activity']['object']['id']]['_nodes'], key=lambda d: d['activity']['object']['published']): _flatten(snode, level=level+1) _flatten(idx[root_id]) @@ -495,10 +495,10 @@ def _build_thread(data, include_children=True): return thread -@app.route('/note/') -def note_by_id(note_id): +@app.route('/note/') +def note_by_id(note_id): data = DB.outbox.find_one({'id': note_id}) - if not data: + if not data: abort(404) if data['meta'].get('deleted', False): abort(410) @@ -511,7 +511,7 @@ def note_by_id(note_id): '$or': [{'activity.object.id': data['activity']['object']['id']}, {'activity.object': data['activity']['object']['id']}], })) - likes = [ACTOR_SERVICE.get(doc['activity']['actor']) for doc in likes] + likes = [ACTOR_SERVICE.get(doc['activity']['actor']) for doc in likes] shares = list(DB.inbox.find({ 'meta.undo': False, @@ -519,7 +519,7 @@ def note_by_id(note_id): '$or': [{'activity.object.id': data['activity']['object']['id']}, {'activity.object': data['activity']['object']['id']}], })) - shares = [ACTOR_SERVICE.get(doc['activity']['actor']) for doc in shares] + shares = [ACTOR_SERVICE.get(doc['activity']['actor']) for doc in shares] return render_template('note.html', likes=likes, shares=shares, me=ME, thread=thread, note=data) @@ -536,7 +536,7 @@ def nodeinfo(): 'openRegistrations': False, 'usage': {'users': {'total': 1}, 'localPosts': DB.outbox.count()}, 'metadata': { - 'sourceCode': 'https://github.com/tsileo/microblog.pub', + 'sourceCode': 'https://github.com/tsileo/microblog.pub', 'nodeName': f'@{USERNAME}@{DOMAIN}', }, }), @@ -551,7 +551,7 @@ def wellknown_nodeinfo(): 'rel': 'http://nodeinfo.diaspora.software/ns/schema/2.0', 'href': f'{ID}/nodeinfo', } - + ], ) @@ -616,11 +616,11 @@ def activity_from_doc(raw_doc: Dict[str, Any], embed: bool = False) -> Dict[str, -@app.route('/outbox', methods=['GET', 'POST']) -def outbox(): - if request.method == 'GET': - if not is_api_request(): - abort(404) +@app.route('/outbox', methods=['GET', 'POST']) +def outbox(): + if request.method == 'GET': + if not is_api_request(): + abort(404) # TODO(tsileo): filter the outbox if not authenticated # FIXME(tsileo): filter deleted, add query support for build_ordered_collection q = { @@ -639,7 +639,7 @@ def outbox(): _api_required() except BadSignature: abort(401) - + data = request.get_json(force=True) print(data) activity = activitypub.parse_activity(data) @@ -785,7 +785,7 @@ def admin(): col_followers=DB.followers.count(), col_following=DB.following.count(), ) - + @app.route('/new', methods=['GET']) @login_required @@ -833,7 +833,7 @@ def notifications(): 'meta.deleted': False, } # TODO(tsileo): also include replies via regex on Create replyTo - q = {'$or': [q, {'type': 'Follow'}, {'type': 'Accept'}, {'type': 'Undo', 'activity.object.type': 'Follow'}, + q = {'$or': [q, {'type': 'Follow'}, {'type': 'Accept'}, {'type': 'Undo', 'activity.object.type': 'Follow'}, {'type': 'Announce', 'activity.object': {'$regex': f'^{BASE_URL}'}}, {'type': 'Create', 'activity.object.inReplyTo': {'$regex': f'^{BASE_URL}'}}, ]} @@ -1004,27 +1004,27 @@ def stream(): ) -@app.route('/inbox', methods=['GET', 'POST']) -def inbox(): - if request.method == 'GET': - if not is_api_request(): - abort(404) +@app.route('/inbox', methods=['GET', 'POST']) +def inbox(): + if request.method == 'GET': + if not is_api_request(): + abort(404) try: _api_required() except BadSignature: abort(404) - return jsonify(**activitypub.build_ordered_collection( - DB.inbox, - q={'meta.deleted': False}, - cursor=request.args.get('cursor'), - map_func=lambda doc: remove_context(doc['activity']), - )) + return jsonify(**activitypub.build_ordered_collection( + DB.inbox, + q={'meta.deleted': False}, + cursor=request.args.get('cursor'), + map_func=lambda doc: remove_context(doc['activity']), + )) - data = request.get_json(force=True) + data = request.get_json(force=True) logger.debug(f'req_headers={request.headers}') logger.debug(f'raw_data={data}') - try: + try: if not verify_request(ACTOR_SERVICE): raise Exception('failed to verify request') except Exception: @@ -1039,13 +1039,13 @@ def inbox(): response=json.dumps({'error': 'failed to verify request (using HTTP signatures or fetching the IRI)'}), ) - activity = activitypub.parse_activity(data) - logger.debug(f'inbox activity={activity}/{data}') - activity.process_from_inbox() + activity = activitypub.parse_activity(data) + logger.debug(f'inbox activity={activity}/{data}') + activity.process_from_inbox() - return Response( - status=201, - ) + return Response( + status=201, + ) @app.route('/api/debug', methods=['GET', 'DELETE']) @@ -1082,17 +1082,17 @@ def api_upload(): print('upload OK') print(filename) attachment = [ - {'mediaType': mtype, - 'name': rfilename, - 'type': 'Document', + {'mediaType': mtype, + 'name': rfilename, + 'type': 'Document', 'url': BASE_URL + f'/static/media/{filename}' }, ] print(attachment) - content = request.args.get('content') - to = request.args.get('to') - note = activitypub.Note( - cc=[ID+'/followers'], + content = request.args.get('content') + to = request.args.get('to') + note = activitypub.Note( + cc=[ID+'/followers'], to=[to if to else config.AS_PUBLIC], content=content, # TODO(tsileo): handle markdown attachment=attachment, @@ -1104,30 +1104,30 @@ def api_upload(): print(create.to_dict()) create.post_to_outbox() print('posted') - + return Response( status=201, response='OK', ) -@app.route('/api/new_note', methods=['POST']) -@api_required -def api_new_note(): +@app.route('/api/new_note', methods=['POST']) +@api_required +def api_new_note(): source = _user_api_arg('content') if not source: raise ValueError('missing content') - + _reply, reply = None, None try: _reply = _user_api_arg('reply') except ValueError: pass - content, tags = parse_markdown(source) + content, tags = parse_markdown(source) to = request.args.get('to') cc = [ID+'/followers'] - + if _reply: reply = activitypub.parse_activity(OBJECT_SERVICE.get(_reply)) cc.append(reply.attributedTo) @@ -1136,8 +1136,8 @@ def api_new_note(): if tag['type'] == 'Mention': cc.append(tag['href']) - note = activitypub.Note( - cc=list(set(cc)), + note = activitypub.Note( + cc=list(set(cc)), to=[to if to else config.AS_PUBLIC], content=content, tag=tags, @@ -1193,20 +1193,20 @@ def api_follow(): return _user_api_response(activity=follow.id) -@app.route('/followers') -def followers(): - if is_api_request(): +@app.route('/followers') +def followers(): + if is_api_request(): return jsonify( **activitypub.build_ordered_collection( DB.followers, cursor=request.args.get('cursor'), map_func=lambda doc: doc['remote_actor'], ) - ) + ) - followers = [ACTOR_SERVICE.get(doc['remote_actor']) for doc in DB.followers.find(limit=50)] - return render_template( - 'followers.html', + followers = [ACTOR_SERVICE.get(doc['remote_actor']) for doc in DB.followers.find(limit=50)] + return render_template( + 'followers.html', me=ME, notes=DB.inbox.find({'object.object.type': 'Note'}).count(), followers=DB.followers.count(), @@ -1225,7 +1225,7 @@ def following(): map_func=lambda doc: doc['remote_actor'], ), ) - + following = [ACTOR_SERVICE.get(doc['remote_actor']) for doc in DB.following.find(limit=50)] return render_template( 'following.html', @@ -1327,13 +1327,13 @@ def get_client_id_data(url): @app.route('/indieauth/flow', methods=['POST']) -@login_required -def indieauth_flow(): - auth = dict( - scope=' '.join(request.form.getlist('scopes')), - me=request.form.get('me'), - client_id=request.form.get('client_id'), - state=request.form.get('state'), +@login_required +def indieauth_flow(): + auth = dict( + scope=' '.join(request.form.getlist('scopes')), + me=request.form.get('me'), + client_id=request.form.get('client_id'), + state=request.form.get('state'), redirect_uri=request.form.get('redirect_uri'), response_type=request.form.get('response_type'), ) @@ -1354,14 +1354,14 @@ def indieauth_flow(): return redirect(red) -# @app.route('/indieauth', methods=['GET', 'POST']) -def indieauth_endpoint(): - if request.method == 'GET': - if not session.get('logged_in'): - return redirect(url_for('login', next=request.url)) +# @app.route('/indieauth', methods=['GET', 'POST']) +def indieauth_endpoint(): + if request.method == 'GET': + if not session.get('logged_in'): + return redirect(url_for('login', next=request.url)) - me = request.args.get('me') - # FIXME(tsileo): ensure me == ID + me = request.args.get('me') + # FIXME(tsileo): ensure me == ID client_id = request.args.get('client_id') redirect_uri = request.args.get('redirect_uri') state = request.args.get('state', '') @@ -1397,7 +1397,7 @@ def indieauth_endpoint(): abort(403) return - session['logged_in'] = True + session['logged_in'] = True me = auth['me'] state = auth['state'] scope = ' '.join(auth['scope']) diff --git a/config.py b/config.py index 506a4d3..9adbff9 100644 --- a/config.py +++ b/config.py @@ -6,10 +6,9 @@ import requests from itsdangerous import JSONWebSignatureSerializer from datetime import datetime -from utils import strtobool -from utils.key import Key, KEY_DIR, get_secret_key -from utils.actor_service import ActorService -from utils.object_service import ObjectService +from little_boxes.utils import strtobool +from utils.key import KEY_DIR, get_key, get_secret_key + def noop(): pass @@ -17,7 +16,7 @@ def noop(): CUSTOM_CACHE_HOOKS = False try: - from cache_hooks import purge as custom_cache_purge_hook + from cache_hooks import purge as custom_cache_purge_hook except ModuleNotFoundError: custom_cache_purge_hook = noop @@ -58,8 +57,6 @@ USER_AGENT = ( f'(microblog.pub/{VERSION}; +{BASE_URL})' ) -# TODO(tsileo): use 'mongo:27017; -# mongo_client = MongoClient(host=['mongo:27017']) mongo_client = MongoClient( host=[os.getenv('MICROBLOGPUB_MONGODB_HOST', 'localhost:27017')], ) @@ -67,23 +64,26 @@ mongo_client = MongoClient( DB_NAME = '{}_{}'.format(USERNAME, DOMAIN.replace('.', '_')) DB = mongo_client[DB_NAME] + def _drop_db(): if not DEBUG_MODE: return mongo_client.drop_database(DB_NAME) -KEY = Key(USERNAME, DOMAIN, create=True) + +KEY = get_key(ID, USERNAME, DOMAIN) JWT_SECRET = get_secret_key('jwt') JWT = JSONWebSignatureSerializer(JWT_SECRET) + def _admin_jwt_token() -> str: return JWT.dumps({'me': 'ADMIN', 'ts': datetime.now().timestamp()}).decode('utf-8') # type: ignore -ADMIN_API_KEY = get_secret_key('admin_api_key', _admin_jwt_token) +ADMIN_API_KEY = get_secret_key('admin_api_key', _admin_jwt_token) ME = { "@context": [ @@ -107,13 +107,5 @@ ME = { "type": "Image", "url": ICON_URL, }, - "publicKey": { - "id": ID+"#main-key", - "owner": ID, - "publicKeyPem": KEY.pubkey_pem, - }, + "publicKey": KEY.to_dict(), } -print(ME) - -ACTOR_SERVICE = ActorService(USER_AGENT, DB.actors_cache, ID, ME, DB.instances) -OBJECT_SERVICE = ObjectService(USER_AGENT, DB.objects_cache, DB.inbox, DB.outbox, DB.instances) diff --git a/little_boxes/README.md b/little_boxes/README.md deleted file mode 100644 index 024e073..0000000 --- a/little_boxes/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# Little Boxes - -Tiny ActivityPub framework written in Python, both database and server agnostic. - -## Getting Started - -```python -from little_boxes import activitypub as ap - -from mydb import db_client - - -class MyBackend(BaseBackend): - - def __init__(self, db_connection): - self.db_connection = db_connection - - def inbox_new(self, as_actor, activity): - # Save activity as "as_actor" - # [...] - - def post_to_remote_inbox(self, as_actor, payload, recipient): - # Send the activity to the remote actor - # [...] - - -db_con = db_client() -my_backend = MyBackend(db_con) - -ap.use_backend(my_backend) - -me = ap.Person({}) # Init an actor -outbox = ap.Outbox(me) - -follow = ap.Follow(actor=me, object='http://iri-i-want-follow') -outbox.post(follow) -``` diff --git a/little_boxes/__init__.py b/little_boxes/__init__.py deleted file mode 100644 index c30c37d..0000000 --- a/little_boxes/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -import logging - -logger = logging.getLogger(__name__) - - -def strtobool(s: str) -> bool: - if s in ['y', 'yes', 'true', 'on', '1']: - return True - if s in ['n', 'no', 'false', 'off', '0']: - return False - - raise ValueError(f'cannot convert {s} to bool') diff --git a/little_boxes/activitypub.py b/little_boxes/activitypub.py deleted file mode 100644 index 781c71e..0000000 --- a/little_boxes/activitypub.py +++ /dev/null @@ -1,1073 +0,0 @@ -"""Core ActivityPub classes.""" -import logging -import json -import binascii -import os -from datetime import datetime -from enum import Enum - -from .errors import BadActivityError -from .errors import UnexpectedActivityTypeError -from .errors import NotFromOutboxError -from .errors import ActivityNotFoundError -from .urlutils import check_url -from .utils import parse_collection - -from typing import List -from typing import Optional -from typing import Dict -from typing import Any -from typing import Union -from typing import Type - -import requests - - -logger = logging.getLogger(__name__) - -# Helper/shortcut for typing -ObjectType = Dict[str, Any] -ObjectOrIDType = Union[str, ObjectType] - -CTX_AS = 'https://www.w3.org/ns/activitystreams' -CTX_SECURITY = 'https://w3id.org/security/v1' -AS_PUBLIC = 'https://www.w3.org/ns/activitystreams#Public' - -COLLECTION_CTX = [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - { - "Hashtag": "as:Hashtag", - "sensitive": "as:sensitive", - } -] - -# Will be used to keep track of all the defined activities -_ACTIVITY_CLS: Dict['ActivityTypeEnum', Type['_BaseActivity']] = {} - -BACKEND = None - -def use_backend(backend_instance): - global BACKEND - BACKEND = backend_instance - - - -class DefaultRemoteObjectFetcher(object): - """Not meant to be used on production, a caching layer, and DB shortcut fox inbox/outbox should be hooked.""" - - def __init__(self): - self._user_agent = 'Little Boxes (+https://github.com/tsileo/little_boxes)' - - def fetch(self, iri): - print('OLD FETCHER') - check_url(iri) - - resp = requests.get(iri, headers={ - 'Accept': 'application/activity+json', - 'User-Agent': self._user_agent, - }) - - if resp.status_code == 404: - raise ActivityNotFoundError(f'{iri} cannot be fetched, 404 not found error') - - resp.raise_for_status() - - return resp.json() - - -class ActivityType(Enum): - """Supported activity `type`.""" - ANNOUNCE = 'Announce' - BLOCK = 'Block' - LIKE = 'Like' - CREATE = 'Create' - UPDATE = 'Update' - PERSON = 'Person' - ORDERED_COLLECTION = 'OrderedCollection' - ORDERED_COLLECTION_PAGE = 'OrderedCollectionPage' - COLLECTION_PAGE = 'CollectionPage' - COLLECTION = 'Collection' - NOTE = 'Note' - ACCEPT = 'Accept' - REJECT = 'Reject' - FOLLOW = 'Follow' - DELETE = 'Delete' - UNDO = 'Undo' - IMAGE = 'Image' - TOMBSTONE = 'Tombstone' - - -def parse_activity(payload: ObjectType, expected: Optional[ActivityType] = None) -> 'BaseActivity': - t = ActivityType(payload['type']) - - if expected and t != expected: - raise UnexpectedActivityTypeError(f'expected a {expected.name} activity, got a {payload["type"]}') - - if t not in _ACTIVITY_CLS: - raise BadActivityError(f'unsupported activity type {payload["type"]}') - - activity = _ACTIVITY_CLS[t](**payload) - - return activity - - -def random_object_id() -> str: - """Generates a random object ID.""" - return binascii.hexlify(os.urandom(8)).decode('utf-8') - - -def _to_list(data: Union[List[Any], Any]) -> List[Any]: - """Helper to convert fields that can be either an object or a list of objects to a list of object.""" - if isinstance(data, list): - return data - return [data] - - -def clean_activity(activity: ObjectType) -> Dict[str, Any]: - """Clean the activity before rendering it. - - Remove the hidden bco and bcc field - """ - for field in ['bto', 'bcc']: - if field in activity: - del(activity[field]) - if activity['type'] == 'Create' and field in activity['object']: - del(activity['object'][field]) - return activity - - -def _get_actor_id(actor: ObjectOrIDType) -> str: - """Helper for retrieving an actor `id`.""" - if isinstance(actor, dict): - return actor['id'] - return actor - - -class BaseBackend(object): - """In-memory backend meant to be used for the test suite.""" - DB = {} - USERS = {} - FETCH_MOCK = {} - INBOX_IDX = {} - OUTBOX_IDX = {} - FOLLOWERS = {} - FOLLOWING = {} - - def setup_actor(self, name, pusername): - """Create a new actor in this backend.""" - p = Person( - name=name, - preferredUsername=pusername, - summary='Hello', - id=f'https://lol.com/{pusername}', - inbox=f'https://lol.com/{pusername}/inbox', - ) - - self.USERS[p.preferredUsername] = p - self.DB[p.id] = { - 'inbox': [], - 'outbox': [], - } - self.INBOX_IDX[p.id] = {} - self.OUTBOX_IDX[p.id] = {} - self.FOLLOWERS[p.id] = [] - self.FOLLOWING[p.id] = [] - self.FETCH_MOCK[p.id] = p.to_dict() - return p - - def fetch_iri(self, iri: str): - return self.FETCH_MOCK[iri] - - def get_user(self, username: str) -> 'Person': - if username in self.USERS: - return self.USERS[username] - else: - raise ValueError(f'bad username {username}') - - def outbox_is_blocked(self, as_actor: 'Person', actor_id: str) -> bool: - """Returns True if `as_actor` has blocked `actor_id`.""" - for activity in self.DB[as_actor.id]['outbox']: - if activity.ACTIVITY_TYPE == ActivityType.BLOCK: - return True - return False - - def inbox_get_by_iri(self, as_actor: 'Person', iri: str) -> Optional['BaseActivity']: - for activity in self.DB[as_actor.id]['inbox']: - if activity.id == iri: - return activity - - return None - - def inbox_new(self, as_actor: 'Person', activity: 'BaseActivity') -> None: - if activity.id in self.INBOX_IDX[as_actor.id]: - return - self.DB[as_actor.id]['inbox'].append(activity) - self.INBOX_IDX[as_actor.id][activity.id] = activity - - def activity_url(self, obj_id: str) -> str: - # from the random hex ID - return 'TODO' - - def outbox_new(self, activity: 'BaseActivity') -> None: - print(f'saving {activity!r} to DB') - actor_id = activity.get_actor().id - if activity.id in self.OUTBOX_IDX[actor_id]: - return - self.DB[actor_id]['outbox'].append(activity) - self.OUTBOX_IDX[actor_id][activity.id] = activity - - def new_follower(self, as_actor: 'Person', actor: 'Person') -> None: - self.FOLLOWERS[as_actor.id].append(actor.id) - - def undo_new_follower(self, actor: 'Person') -> None: - pass - - def new_following(self, as_actor: 'Person', actor: 'Person') -> None: - print(f'new following {actor!r}') - self.FOLLOWING[as_actor.id].append(actor.id) - - def undo_new_following(self, actor: 'Person') -> None: - pass - - def followers(self, as_actor: 'Person') -> List[str]: - return self.FOLLOWERS[as_actor.id] - - def following(self, as_actor: 'Person') -> List[str]: - return self.FOLLOWING[as_actor.id] - - def post_to_remote_inbox(self, payload_encoded: str, recp: str) -> None: - payload = json.loads(payload_encoded) - print(f'post_to_remote_inbox {payload} {recp}') - act = parse_activity(payload) - as_actor = parse_activity(self.fetch_iri(recp.replace('/inbox', ''))) - act.process_from_inbox(as_actor) - - def is_from_outbox(self, activity: 'BaseActivity') -> None: - pass - - def inbox_like(self, activity: 'Like') -> None: - pass - - def inbox_undo_like(self, activity: 'Like') -> None: - pass - - def outbox_like(self, activity: 'Like') -> None: - pass - - def outbox_undo_like(self, activity: 'Lke') -> None: - pass - - def inbox_announce(self, activity: 'Announce') -> None: - pass - - def inbox_undo_announce(self, activity: 'Announce') -> None: - pass - - def outbox_announce(self, activity: 'Announce') -> None: - pass - - def outbox_undo_announce(self, activity: 'Announce') -> None: - pass - - def inbox_delete(self, activity: 'Delete') -> None: - pass - - def outbox_delete(self, activity: 'Delete') -> None: - pass - - def inbox_update(self, activity: 'Update') -> None: - pass - - def outbox_update(self, activity: 'Update') -> None: - pass - - def inbox_create(self, activity: 'Create') -> None: - pass - - def outbox_create(self, activity: 'Create') -> None: - pass - - -class _ActivityMeta(type): - """Metaclass for keeping track of subclass.""" - def __new__(meta, name, bases, class_dict): - cls = type.__new__(meta, name, bases, class_dict) - - # Ensure the class has an activity type defined - if name != 'BaseActivity' and not cls.ACTIVITY_TYPE: - raise ValueError(f'class {name} has no ACTIVITY_TYPE') - - # Register it - _ACTIVITY_CLS[cls.ACTIVITY_TYPE] = cls - return cls - - -class BaseActivity(object, metaclass=_ActivityMeta): - """Base class for ActivityPub activities.""" - - ACTIVITY_TYPE: Optional[ActivityType] = None # the ActivityTypeEnum the class will represent - OBJECT_REQUIRED = False # Whether the object field is required or note - ALLOWED_OBJECT_TYPES: List[ActivityType] = [] - ACTOR_REQUIRED = True # Most of the object requires an actor, so this flag in on by default - - def __init__(self, **kwargs) -> None: - if kwargs.get('type') and kwargs.pop('type') != self.ACTIVITY_TYPE.value: - raise UnexpectedActivityTypeError(f'Expect the type to be {self.ACTIVITY_TYPE.value!r}') - - # Initialize the dict that will contains all the activity fields - self._data: Dict[str, Any] = { - 'type': self.ACTIVITY_TYPE.value - } - logger.debug(f'initializing a {self.ACTIVITY_TYPE.value} activity: {kwargs!r}') - - # A place to set ephemeral data - self.__ctx = {} - - # The id may not be present for new activities - if 'id' in kwargs: - self._data['id'] = kwargs.pop('id') - - if self.ACTIVITY_TYPE != ActivityType.PERSON and self.ACTOR_REQUIRED: - actor = kwargs.get('actor') - if actor: - kwargs.pop('actor') - actor = self._validate_person(actor) - self._data['actor'] = actor - else: - raise BadActivityError('missing actor') - - if self.OBJECT_REQUIRED and 'object' in kwargs: - obj = kwargs.pop('object') - if isinstance(obj, str): - # The object is a just a reference the its ID/IRI - # FIXME(tsileo): fetch the ref - self._data['object'] = obj - else: - if not self.ALLOWED_OBJECT_TYPES: - raise UnexpectedActivityTypeError('unexpected object') - if 'type' not in obj or (self.ACTIVITY_TYPE != ActivityType.CREATE and 'id' not in obj): - raise BadActivityError('invalid object, missing type') - if ActivityType(obj['type']) not in self.ALLOWED_OBJECT_TYPES: - raise UnexpectedActivityTypeError( - f'unexpected object type {obj["type"]} (allowed={self.ALLOWED_OBJECT_TYPES!r})' - ) - self._data['object'] = obj - - if '@context' not in kwargs: - self._data['@context'] = CTX_AS - else: - self._data['@context'] = kwargs.pop('@context') - - # @context check - if not isinstance(self._data['@context'], list): - self._data['@context'] = [self._data['@context']] - if CTX_SECURITY not in self._data['@context']: - self._data['@context'].append(CTX_SECURITY) - if isinstance(self._data['@context'][-1], dict): - self._data['@context'][-1]['Hashtag'] = 'as:Hashtag' - self._data['@context'][-1]['sensitive'] = 'as:sensitive' - else: - self._data['@context'].append({'Hashtag': 'as:Hashtag', 'sensitive': 'as:sensitive'}) - - # FIXME(tsileo): keys required for some subclasses? - allowed_keys = None - try: - allowed_keys = self._init(**kwargs) - logger.debug('calling custom init') - except NotImplementedError: - pass - - if allowed_keys: - # Allows an extra to (like for Accept and Follow) - kwargs.pop('to', None) - if len(set(kwargs.keys()) - set(allowed_keys)) > 0: - raise BadActivityError(f'extra data left: {kwargs!r}') - else: - # Remove keys with `None` value - valid_kwargs = {} - for k, v in kwargs.items(): - if v is None: - continue - valid_kwargs[k] = v - self._data.update(**valid_kwargs) - - def ctx(self) -> Dict[str, Any]: - return self.__ctx - - def set_ctx(self, ctx: Dict[str, Any]) -> None: - self.__ctx = ctx - - def _init(self, **kwargs) -> Optional[List[str]]: - """Optional init callback that may returns a list of allowed keys.""" - raise NotImplementedError - - def __repr__(self) -> str: - """Pretty repr.""" - return '{}({!r})'.format(self.__class__.__qualname__, self._data.get('id')) - - def __str__(self) -> str: - """Returns the ID/IRI when castign to str.""" - return str(self._data.get('id', f'[new {self.ACTIVITY_TYPE} activity]')) - - def __getattr__(self, name: str) -> Any: - """Allow to access the object field as regular attributes.""" - if self._data.get(name): - return self._data.get(name) - - def _outbox_set_id(self, uri: str, obj_id: str) -> None: - """Optional callback for subclasses to so something with a newly generated ID (for outbox activities).""" - raise NotImplementedError - - def outbox_set_id(self, uri: str, obj_id: str) -> None: - """Set the ID for a new activity.""" - logger.debug(f'setting ID {uri} / {obj_id}') - self._data['id'] = uri - try: - self._outbox_set_id(uri, obj_id) - except NotImplementedError: - pass - - def _actor_id(self, obj: ObjectOrIDType) -> str: - if isinstance(obj, dict) and obj['type'] == ActivityType.PERSON.value: - obj_id = obj.get('id') - if not obj_id: - raise BadActivityError(f'missing object id: {obj!r}') - return obj_id - elif isinstance(obj, str): - return obj - else: - raise BadActivityError(f'invalid "actor" field: {obj!r}') - - def _validate_person(self, obj: ObjectOrIDType) -> str: - obj_id = self._actor_id(obj) - try: - actor = BACKEND.fetch_iri(obj_id) - except Exception: - raise BadActivityError(f'failed to validate actor {obj!r}') - - if not actor or 'id' not in actor: - raise BadActivityError(f'invalid actor {actor}') - - return actor['id'] - - def get_object(self) -> 'BaseActivity': - """Returns the object as a BaseActivity instance.""" - if self.__obj: - return self.__obj - if isinstance(self._data['object'], dict): - p = parse_activity(self._data['object']) - else: - obj = BACKEND.fetch_iri(self._data['object']) - if ActivityType(obj.get('type')) not in self.ALLOWED_OBJECT_TYPES: - raise UnexpectedActivityTypeError(f'invalid object type {obj.get("type")!r}') - p = parse_activity(obj) - - self.__obj: Optional['BaseActivity'] = p - return p - - def reset_object_cache(self) -> None: - self.__obj = None - - def to_dict(self, embed: bool = False, embed_object_id_only: bool = False) -> ObjectType: - """Serializes the activity back to a dict, ready to be JSON serialized.""" - data = dict(self._data) - if embed: - for k in ['@context', 'signature']: - if k in data: - del(data[k]) - if data.get('object') and embed_object_id_only and isinstance(data['object'], dict): - try: - data['object'] = data['object']['id'] - except KeyError: - raise BadActivityError(f'embedded object {data["object"]!r} should have an id') - - return data - - def get_actor(self) -> 'BaseActivity': - # FIXME(tsileo): cache the actor (same way as get_object) - actor = self._data.get('actor') - if not actor and self.ACTOR_REQUIRED: - # Quick hack for Note objects - if self.ACTIVITY_TYPE == ActivityType.NOTE: - actor = str(self._data.get('attributedTo')) - else: - raise BadActivityError(f'failed to fetch actor: {self._data!r}') - - actor_id = self._actor_id(actor) - return Person(**BACKEND.fetch_iri(actor_id)) - - def _pre_post_to_outbox(self) -> None: - raise NotImplementedError - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - raise NotImplementedError - - def _undo_outbox(self) -> None: - raise NotImplementedError - - def _pre_process_from_inbox(self, as_actor: 'Person') -> None: - raise NotImplementedError - - def _process_from_inbox(self, as_actor: 'Person') -> None: - raise NotImplementedError - - def _undo_inbox(self) -> None: - raise NotImplementedError - - def process_from_inbox(self, as_actor: 'Person') -> None: - """Process the message posted to `as_actor` inbox.""" - logger.debug(f'calling main process from inbox hook for {self}') - actor = self.get_actor() - - # Check for Block activity - if BACKEND.outbox_is_blocked(as_actor, actor.id): - # TODO(tsileo): raise ActorBlockedError? - logger.info(f'actor {actor!r} is blocked, dropping the received activity {self!r}') - return - - if BACKEND.inbox_get_by_iri(as_actor, self.id): - # The activity is already in the inbox - logger.info(f'received duplicate activity {self}, dropping it') - return - - try: - self._pre_process_from_inbox(as_actor) - logger.debug('called pre process from inbox hook') - except NotImplementedError: - logger.debug('pre process from inbox hook not implemented') - - BACKEND.inbox_new(as_actor, self) - logger.info('activity {self!r} saved') - - try: - self._process_from_inbox(as_actor) - logger.debug('called process from inbox hook') - except NotImplementedError: - logger.debug('process from inbox hook not implemented') - - def post_to_outbox(self) -> None: - logger.debug(f'calling main post to outbox hook for {self}') - - # Assign create a random ID - obj_id = random_object_id() - self.outbox_set_id(BACKEND.activity_url(obj_id), obj_id) - - try: - self._pre_post_to_outbox() - logger.debug(f'called pre post to outbox hook') - except NotImplementedError: - logger.debug('pre post to outbox hook not implemented') - - BACKEND.outbox_new(self) - - recipients = self.recipients() - logger.info(f'recipients={recipients}') - activity = clean_activity(self.to_dict()) - - try: - self._post_to_outbox(obj_id, activity, recipients) - logger.debug(f'called post to outbox hook') - except NotImplementedError: - logger.debug('post to outbox hook not implemented') - - payload = json.dumps(activity) - for recp in recipients: - logger.debug(f'posting to {recp}') - - BACKEND.post_to_remote_inbox(payload, recp) - - def _recipients(self) -> List[str]: - return [] - - def recipients(self) -> List[str]: - recipients = self._recipients() - actor_id = self.get_actor().id - - out: List[str] = [] - for recipient in recipients: - # if recipient in PUBLIC_INSTANCES: - # if recipient not in out: - # out.append(str(recipient)) - # continue - if recipient in [actor_id, AS_PUBLIC, None]: - continue - if isinstance(recipient, Person): - if recipient.id == actor_id: - continue - actor = recipient - else: - raw_actor = BACKEND.fetch_iri(recipient) - if raw_actor['type'] == ActivityType.PERSON.value: - actor = Person(**raw_actor) - - if actor.endpoints: - shared_inbox = actor.endpoints.get('sharedInbox') - if shared_inbox not in out: - out.append(shared_inbox) - continue - - if actor.inbox and actor.inbox not in out: - out.append(actor.inbox) - - # Is the activity a `Collection`/`OrderedCollection`? - elif raw_actor['type'] in [ActivityType.COLLECTION.value, - ActivityType.ORDERED_COLLECTION.value]: - for item in parse_collection(raw_actor): - if item in [actor_id, AS_PUBLIC]: - continue - try: - col_actor = Person(**BACKEND.fetch_iri(item)) - except UnexpectedActivityTypeError: - logger.exception(f'failed to fetch actor {item!r}') - - if col_actor.endpoints: - shared_inbox = col_actor.endpoints.get('sharedInbox') - if shared_inbox not in out: - out.append(shared_inbox) - continue - if col_actor.inbox and col_actor.inbox not in out: - out.append(col_actor.inbox) - else: - raise BadActivityError(f'failed to parse {raw_actor!r}') - - return out - - def build_undo(self) -> 'BaseActivity': - raise NotImplementedError - - def build_delete(self) -> 'BaseActivity': - raise NotImplementedError - - -class Person(BaseActivity): - ACTIVITY_TYPE = ActivityType.PERSON - OBJECT_REQUIRED = False - ACTOR_REQUIRED = False - - -class Block(BaseActivity): - ACTIVITY_TYPE = ActivityType.BLOCK - OBJECT_REQUIRED = True - ACTOR_REQUIRED = True - - -class Collection(BaseActivity): - ACTIVITY_TYPE = ActivityType.COLLECTION - OBJECT_REQUIRED = False - ACTOR_REQUIRED = False - - -class Image(BaseActivity): - ACTIVITY_TYPE = ActivityType.IMAGE - OBJECT_REQUIRED = False - ACTOR_REQUIRED = False - - def _init(self, **kwargs): - self._data.update( - url=kwargs.pop('url'), - ) - - def __repr__(self): - return 'Image({!r})'.format(self._data.get('url')) - - -class Follow(BaseActivity): - ACTIVITY_TYPE = ActivityType.FOLLOW - ALLOWED_OBJECT_TYPES = [ActivityType.PERSON] - OBJECT_REQUIRED = True - ACTOR_REQUIRED = True - - def _build_reply(self, reply_type: ActivityType) -> BaseActivity: - if reply_type == ActivityType.ACCEPT: - return Accept( - actor=self.get_object().id, - object=self.to_dict(embed=True), - ) - - raise ValueError(f'type {reply_type} is invalid for building a reply') - - def _recipients(self) -> List[str]: - return [self.get_object().id] - - def _process_from_inbox(self, as_actor: 'Person') -> None: - """Receiving a Follow should trigger an Accept.""" - accept = self.build_accept() - accept.post_to_outbox() - - remote_actor = self.get_actor() - - # ABC - BACKEND.new_follower(as_actor, remote_actor) - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - # XXX The new_following event will be triggered by Accept - pass - - def _undo_inbox(self) -> None: - # ABC - self.undo_new_follower(self.get_actor()) - - def _undo_outbox(self) -> None: - # ABC - self.undo_new_following(self.get_actor()) - - def build_accept(self) -> BaseActivity: - return self._build_reply(ActivityType.ACCEPT) - - def build_undo(self) -> BaseActivity: - return Undo(object=self.to_dict(embed=True)) - - -class Accept(BaseActivity): - ACTIVITY_TYPE = ActivityType.ACCEPT - ALLOWED_OBJECT_TYPES = [ActivityType.FOLLOW] - OBJECT_REQUIRED = True - ACTOR_REQUIRED = True - - def _recipients(self) -> List[str]: - return [self.get_object().get_actor().id] - - def _pre_process_from_inbox(self, as_actor: 'Person') -> None: - # FIXME(tsileo): ensure the actor match the object actor - pass - - def _process_from_inbox(self, as_actor: 'Person') -> None: - BACKEND.new_following(as_actor, self.get_actor()) - - -class Undo(BaseActivity): - ACTIVITY_TYPE = ActivityType.UNDO - ALLOWED_OBJECT_TYPES = [ActivityType.FOLLOW, ActivityType.LIKE, ActivityType.ANNOUNCE] - OBJECT_REQUIRED = True - ACTOR_REQUIRED = True - - def _recipients(self) -> List[str]: - obj = self.get_object() - if obj.type_enum == ActivityType.FOLLOW: - return [obj.get_object().id] - else: - return [obj.get_object().get_actor().id] - # TODO(tsileo): handle like and announce - raise Exception('TODO') - - def _pre_process_from_inbox(self, as_actor: 'Person') -> None: - """Ensures an Undo activity comes from the same actor as the updated activity.""" - obj = self.get_object() - actor = self.get_actor() - if actor.id != obj.get_actor().id: - raise BadActivityError(f'{actor!r} cannot update {obj!r}') - - def _process_from_inbox(self, as_actor: 'Person') -> None: - obj = self.get_object() - # FIXME(tsileo): move this to _undo_inbox impl - # DB.inbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) - - try: - obj._undo_inbox() - except NotImplementedError: - pass - - def _pre_post_to_outbox(self) -> None: - """Ensures an Undo activity references an activity owned by the instance.""" - # ABC - if not self.is_from_outbox(self): - raise NotFromOutboxError(f'object {self!r} is not owned by this instance') - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - logger.debug('processing undo to outbox') - logger.debug('self={}'.format(self)) - obj = self.get_object() - logger.debug('obj={}'.format(obj)) - - # FIXME(tsileo): move this to _undo_inbox impl - # DB.outbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) - - try: - obj._undo_outbox() - logger.debug(f'_undo_outbox called for {obj}') - except NotImplementedError: - logger.debug(f'_undo_outbox not implemented for {obj}') - pass - - -class Like(BaseActivity): - ACTIVITY_TYPE = ActivityType.LIKE - ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] - OBJECT_REQUIRED = True - ACTOR_REQUIRED = True - - def _recipients(self) -> List[str]: - return [self.get_object().get_actor().id] - - def _process_from_inbox(self, as_actor: 'Person') -> None: - # ABC - self.inbox_like(self) - - def _undo_inbox(self) -> None: - # ABC - self.inbox_undo_like(self) - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]): - # ABC - self.outbox_like(self) - - def _undo_outbox(self) -> None: - # ABC - self.outbox_undo_like(self) - - def build_undo(self) -> BaseActivity: - return Undo( - object=self.to_dict(embed=True, embed_object_id_only=True), - actor=self.get_actor().id, - ) - - -class Announce(BaseActivity): - ACTIVITY_TYPE = ActivityType.ANNOUNCE - ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] - OBJECT_REQUIRED = True - ACTOR_REQUIRED = True - - def _recipients(self) -> List[str]: - recipients = [] - - for field in ['to', 'cc']: - if field in self._data: - recipients.extend(_to_list(self._data[field])) - - return recipients - - def _process_from_inbox(self, as_actor: 'Person') -> None: - if isinstance(self._data['object'], str) and not self._data['object'].startswith('http'): - # TODO(tsileo): actually drop it without storing it and better logging, also move the check somewhere else - logger.warn( - f'received an Annouce referencing an OStatus notice ({self._data["object"]}), dropping the message' - ) - return - - # ABC - self.inbox_announce(self) - - def _undo_inbox(self) -> None: - # ABC - self.inbox_undo_annnounce(self) - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - # ABC - self.outbox_announce(self) - - def _undo_outbox(self) -> None: - # ABC - self.outbox_undo_announce(self) - - def build_undo(self) -> BaseActivity: - return Undo(object=self.to_dict(embed=True)) - - -class Delete(BaseActivity): - ACTIVITY_TYPE = ActivityType.DELETE - ALLOWED_OBJECT_TYPES = [ActivityType.NOTE, ActivityType.TOMBSTONE] - OBJECT_REQUIRED = True - - def _get_actual_object(self) -> BaseActivity: - # FIXME(tsileo): overrides get_object instead? - obj = self.get_object() - if obj.type_enum == ActivityType.TOMBSTONE: - obj = parse_activity(BACKEND.fetch_iri(obj.id)) - return obj - - def _recipients(self) -> List[str]: - obj = self._get_actual_object() - return obj._recipients() - - def _pre_process_from_inbox(self, as_actor: 'Person') -> None: - """Ensures a Delete activity comes from the same actor as the deleted activity.""" - obj = self._get_actual_object() - actor = self.get_actor() - if actor.id != obj.get_actor().id: - raise BadActivityError(f'{actor!r} cannot delete {obj!r}') - - def _process_from_inbox(self, as_actor: 'Person') -> None: - # ABC - self.inbox_delete(self) - # FIXME(tsileo): handle the delete_threads here? - - def _pre_post_to_outbox(self) -> None: - """Ensures the Delete activity references a activity from the outbox (i.e. owned by the instance).""" - obj = self._get_actual_object() - # ABC - if not self.is_from_outbox(self): - raise NotFromOutboxError(f'object {obj["id"]} is not owned by this instance') - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - # ABC - self.outbox_delete(self) - - -class Update(BaseActivity): - ACTIVITY_TYPE = ActivityType.UPDATE - ALLOWED_OBJECT_TYPES = [ActivityType.NOTE, ActivityType.PERSON] - OBJECT_REQUIRED = True - ACTOR_REQUIRED = True - - def _pre_process_from_inbox(self, as_actor: 'Person') -> None: - """Ensures an Update activity comes from the same actor as the updated activity.""" - obj = self.get_object() - actor = self.get_actor() - if actor.id != obj.get_actor().id: - raise BadActivityError(f'{actor!r} cannot update {obj!r}') - - def _process_from_inbox(self, as_actor: 'Person') -> None: - # ABC - self.inbox_update(self) - - def _pre_post_to_outbox(self) -> None: - # ABC - if not self.is_form_outbox(self): - raise NotFromOutboxError(f'object {self!r} is not owned by this instance') - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - # ABC - self.outbox_update(self) - - -class Create(BaseActivity): - ACTIVITY_TYPE = ActivityType.CREATE - ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] - OBJECT_REQUIRED = True - ACTOR_REQUIRED = True - - def _set_id(self, uri: str, obj_id: str) -> None: - self._data['object']['id'] = uri + '/activity' - # ABC - self._data['object']['url'] = self.note_url(self) - self.reset_object_cache() - - def _init(self, **kwargs): - obj = self.get_object() - if not obj.attributedTo: - self._data['object']['attributedTo'] = self.get_actor().id - if not obj.published: - if self.published: - self._data['object']['published'] = self.published - else: - now = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' - self._data['published'] = now - self._data['object']['published'] = now - - def _recipients(self) -> List[str]: - # TODO(tsileo): audience support? - recipients = [] - for field in ['to', 'cc', 'bto', 'bcc']: - if field in self._data: - recipients.extend(_to_list(self._data[field])) - - recipients.extend(self.get_object()._recipients()) - - return recipients - - def _process_from_inbox(self, as_actor: 'Person') -> None: - # ABC - self.inbox_create(self) - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - # ABC - self.outbox_create(self) - - -class Tombstone(BaseActivity): - ACTIVITY_TYPE = ActivityType.TOMBSTONE - ACTOR_REQUIRED = False - OBJECT_REQUIRED = False - - -class Note(BaseActivity): - ACTIVITY_TYPE = ActivityType.NOTE - ACTOR_REQUIRED = True - OBJECT_REQURIED = False - - def _init(self, **kwargs): - print(self._data) - # Remove the `actor` field as `attributedTo` is used for `Note` instead - if 'actor' in self._data: - del(self._data['actor']) - if 'sensitive' not in kwargs: - self._data['sensitive'] = False - - def _recipients(self) -> List[str]: - # TODO(tsileo): audience support? - recipients: List[str] = [] - - # FIXME(tsileo): re-add support for the PUBLIC_INSTANCES - # If the note is public, we publish it to the defined "public instances" - # if AS_PUBLIC in self._data.get('to', []): - # recipients.extend(PUBLIC_INSTANCES) - # print('publishing to public instances') - # print(recipients) - - for field in ['to', 'cc', 'bto', 'bcc']: - if field in self._data: - recipients.extend(_to_list(self._data[field])) - - return recipients - - def build_create(self) -> BaseActivity: - """Wraps an activity in a Create activity.""" - create_payload = { - 'object': self.to_dict(embed=True), - 'actor': self.attributedTo, - } - for field in ['published', 'to', 'bto', 'cc', 'bcc', 'audience']: - if field in self._data: - create_payload[field] = self._data[field] - - return Create(**create_payload) - - def build_like(self) -> BaseActivity: - return Like(object=self.id) - - def build_announce(self) -> BaseActivity: - return Announce( - object=self.id, - to=[AS_PUBLIC], - cc=[self.follower_collection_id(self.get_actor()), self.attributedTo], # ABC - published=datetime.utcnow().replace(microsecond=0).isoformat() + 'Z', - ) - - def build_delete(self) -> BaseActivity: - return Delete(object=Tombstone(id=self.id).to_dict(embed=True)) - - def get_tombstone(self, deleted: Optional[str]) -> BaseActivity: - return Tombstone( - id=self.id, - published=self.published, - deleted=deleted, - updated=deleted, - ) - - -class Box(object): - def __init__(self, actor: Person): - self.actor = actor - - -class Outbox(Box): - - def post(self, activity: BaseActivity) -> None: - if activity.get_actor().id != self.actor.id: - raise ValueError(f'{activity.get_actor()!r} cannot post into {self.actor!r} outbox') - - activity.post_to_outbox() - - def get(self, activity_iri: str) -> BaseActivity: - pass - - def collection(self): - # TODO(tsileo): figure out an API - pass - - -class Inbox(Box): - - def post(self, activity: BaseActivity) -> None: - activity.process_from_inbox(self.actor) diff --git a/little_boxes/errors.py b/little_boxes/errors.py deleted file mode 100644 index cb3cb34..0000000 --- a/little_boxes/errors.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Errors raised by this package.""" -from typing import Optional -from typing import Dict -from typing import Any - - -class Error(Exception): - """HTTP-friendly base error, with a status code, a message and an optional payload.""" - status_code = 400 - - def __init__( - self, message: str, - status_code: Optional[int] = None, - payload: Optional[Dict[str, Any]] = None, - ) -> None: - Exception.__init__(self) - self.message = message - if status_code is not None: - self.status_code = status_code - self.payload = payload - - def to_dict(self) -> Dict[str, Any]: - rv = dict(self.payload or ()) - rv['message'] = self.message - return rv - - def __repr__(self) -> str: - return ( - f'{self.__class__.__qualname__}({self.message!r}, payload={self.payload!r}, status_code={self.status_code})' - ) - - -class ActorBlockedError(Error): - """Raised when an activity from a blocked actor is received.""" - - -class NotFromOutboxError(Error): - """Raised when an activity targets an object from the inbox when an object from the oubox was expected.""" - - -class ActivityNotFoundError(Error): - """Raised when an activity is not found.""" - status_code = 404 - - -class BadActivityError(Error): - """Raised when an activity could not be parsed/initialized.""" - - -class RecursionLimitExceededError(BadActivityError): - """Raised when the recursion limit for fetching remote object was exceeded (likely a collection).""" - - -class UnexpectedActivityTypeError(BadActivityError): - """Raised when an another activty was expected.""" diff --git a/little_boxes/urlutils.py b/little_boxes/urlutils.py deleted file mode 100644 index 99f900d..0000000 --- a/little_boxes/urlutils.py +++ /dev/null @@ -1,47 +0,0 @@ -import logging -import os -import socket -import ipaddress -from urllib.parse import urlparse - -from . import strtobool -from .errors import Error - -logger = logging.getLogger(__name__) - - -class InvalidURLError(Error): - pass - - -def is_url_valid(url: str) -> bool: - parsed = urlparse(url) - if parsed.scheme not in ['http', 'https']: - return False - - # XXX in debug mode, we want to allow requests to localhost to test the federation with local instances - debug_mode = strtobool(os.getenv('MICROBLOGPUB_DEBUG', 'false')) - if debug_mode: - return True - - if parsed.hostname in ['localhost']: - return False - - try: - ip_address = socket.getaddrinfo(parsed.hostname, parsed.port or 80)[0][4][0] - except socket.gaierror: - logger.exception(f'failed to lookup url {url}') - return False - - if ipaddress.ip_address(ip_address).is_private: - logger.info(f'rejecting private URL {url}') - return False - - return True - - -def check_url(url: str) -> None: - if not is_url_valid(url): - raise InvalidURLError(f'"{url}" is invalid') - - return None diff --git a/little_boxes/utils.py b/little_boxes/utils.py deleted file mode 100644 index 2476182..0000000 --- a/little_boxes/utils.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Contains some ActivityPub related utils.""" -from typing import Optional -from typing import Callable -from typing import Dict -from typing import List -from typing import Any - - -from .errors import RecursionLimitExceededError -from .errors import UnexpectedActivityTypeError - - -def parse_collection( - payload: Optional[Dict[str, Any]] = None, - url: Optional[str] = None, - level: int = 0, - fetcher: Optional[Callable[[str], Dict[str, Any]]] = None, -) -> List[Any]: - """Resolve/fetch a `Collection`/`OrderedCollection`.""" - if not fetcher: - raise Exception('must provide a fetcher') - if level > 3: - raise RecursionLimitExceededError('recursion limit exceeded') - - # Go through all the pages - out: List[Any] = [] - if url: - payload = fetcher(url) - if not payload: - raise ValueError('must at least prove a payload or an URL') - - if payload['type'] in ['Collection', 'OrderedCollection']: - if 'orderedItems' in payload: - return payload['orderedItems'] - if 'items' in payload: - return payload['items'] - if 'first' in payload: - if 'orderedItems' in payload['first']: - out.extend(payload['first']['orderedItems']) - if 'items' in payload['first']: - out.extend(payload['first']['items']) - n = payload['first'].get('next') - if n: - out.extend(parse_collection(url=n, level=level+1, fetcher=fetcher)) - return out - - while payload: - if payload['type'] in ['CollectionPage', 'OrderedCollectionPage']: - if 'orderedItems' in payload: - out.extend(payload['orderedItems']) - if 'items' in payload: - out.extend(payload['items']) - n = payload.get('next') - if n is None: - break - payload = fetcher(n) - else: - raise UnexpectedActivityTypeError('unexpected activity type {}'.format(payload['type'])) - - return out diff --git a/requirements.txt b/requirements.txt index 425405f..eb16141 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,21 +2,19 @@ libsass gunicorn piexif requests -markdown python-u2flib-server Flask Flask-WTF Celery pymongo -pyld timeago bleach -pycryptodome html2text feedgen itsdangerous bcrypt mf2py passlib -pyyaml git+https://github.com/erikriver/opengraph.git +git+https://github.com/tsileo/little-boxes.git +pyyaml diff --git a/test_little_boxes.py b/test_little_boxes.py deleted file mode 100644 index b297c8f..0000000 --- a/test_little_boxes.py +++ /dev/null @@ -1,26 +0,0 @@ -from little_boxes.activitypub import use_backend -from little_boxes.activitypub import BaseBackend -from little_boxes.activitypub import Outbox -from little_boxes.activitypub import Person -from little_boxes.activitypub import Follow - -def test_little_boxes_follow(): - back = BaseBackend() - use_backend(back) - - me = back.setup_actor('Thomas', 'tom') - - other = back.setup_actor('Thomas', 'tom2') - - outbox = Outbox(me) - f = Follow( - actor=me.id, - object=other.id, - ) - - outbox.post(f) - assert back.followers(other) == [me.id] - assert back.following(other) == [] - - assert back.followers(me) == [] - assert back.following(me) == [other.id] diff --git a/utils/key.py b/utils/key.py index f5a2455..18162a5 100644 --- a/utils/key.py +++ b/utils/key.py @@ -1,22 +1,23 @@ import os import binascii -from Crypto.PublicKey import RSA from typing import Callable -KEY_DIR = os.path.join( - os.path.dirname(os.path.abspath(__file__)), '..', 'config' -) +from little_boxes.key import Key + +KEY_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "config") def _new_key() -> str: - return binascii.hexlify(os.urandom(32)).decode('utf-8') + return binascii.hexlify(os.urandom(32)).decode("utf-8") + def get_secret_key(name: str, new_key: Callable[[], str] = _new_key) -> str: - key_path = os.path.join(KEY_DIR, f'{name}.key') + """Loads or generates a cryptographic key.""" + key_path = os.path.join(KEY_DIR, f"{name}.key") if not os.path.exists(key_path): k = new_key() - with open(key_path, 'w+') as f: + with open(key_path, "w+") as f: f.write(k) return k @@ -24,23 +25,19 @@ def get_secret_key(name: str, new_key: Callable[[], str] = _new_key) -> str: return f.read() -class Key(object): - DEFAULT_KEY_SIZE = 2048 - def __init__(self, user: str, domain: str, create: bool = True) -> None: - user = user.replace('.', '_') - domain = domain.replace('.', '_') - key_path = os.path.join(KEY_DIR, f'key_{user}_{domain}.pem') - if os.path.isfile(key_path): - with open(key_path) as f: - self.privkey_pem = f.read() - self.privkey = RSA.importKey(self.privkey_pem) - self.pubkey_pem = self.privkey.publickey().exportKey('PEM').decode('utf-8') - else: - if not create: - raise Exception('must init private key first') - k = RSA.generate(self.DEFAULT_KEY_SIZE) - self.privkey_pem = k.exportKey('PEM').decode('utf-8') - self.pubkey_pem = k.publickey().exportKey('PEM').decode('utf-8') - with open(key_path, 'w') as f: - f.write(self.privkey_pem) - self.privkey = k +def get_key(owner: str, user: str, domain: str) -> Key: + """"Loads or generates an RSA key.""" + k = Key(owner) + user = user.replace(".", "_") + domain = domain.replace(".", "_") + key_path = os.path.join(KEY_DIR, f"key_{user}_{domain}.pem") + if os.path.isfile(key_path): + with open(key_path) as f: + privkey_pem = f.read() + k.load(privkey_pem) + else: + k.new() + with open(key_path, "w") as f: + f.write(k.privkey_pem) + + return k From d9362adb25cd988962269632cd6de8e963a82944 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 16 Jun 2018 22:02:10 +0200 Subject: [PATCH 0112/1425] Black/isort --- activitypub.py | 26 +- app.py | 1295 +++++++++++++++++++----------------- config.py | 17 +- dev-requirements.txt | 1 + tasks.py | 58 +- tests/federation_test.py | 8 +- utils/activitypub_utils.py | 5 +- utils/actor_service.py | 4 +- utils/content_helper.py | 25 +- utils/httpsig.py | 13 +- utils/key.py | 3 +- utils/linked_data_sig.py | 11 +- utils/object_service.py | 5 +- utils/opengraph.py | 5 +- utils/urlutils.py | 2 +- utils/webfinger.py | 8 +- 16 files changed, 814 insertions(+), 672 deletions(-) diff --git a/activitypub.py b/activitypub.py index 697ec59..9097d05 100644 --- a/activitypub.py +++ b/activitypub.py @@ -1,23 +1,30 @@ import logging - from datetime import datetime +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Union from bson.objectid import ObjectId -from html2text import html2text from feedgen.feed import FeedGenerator +from html2text import html2text +import tasks +from config import BASE_URL +from config import DB +from config import ID +from config import ME +from config import USER_AGENT +from config import USERNAME from little_boxes import activitypub as ap from little_boxes.backend import Backend from little_boxes.collection import parse_collection as ap_parse_collection -from config import USERNAME, BASE_URL, ID -from config import DB, ME -import tasks - -from typing import List, Optional, Dict, Any, Union - logger = logging.getLogger(__name__) +MY_PERSON = ap.Person(**ME) + def _remove_id(doc: ap.ObjectType) -> ap.ObjectType: """Helper for removing MongoDB's `_id` field.""" @@ -35,6 +42,9 @@ def _to_list(data: Union[List[Any], Any]) -> List[Any]: class MicroblogPubBackend(Backend): + def user_agent(self) -> str: + return USER_AGENT + def base_url(self) -> str: return BASE_URL diff --git a/app.py b/app.py index 6b104d2..e337204 100644 --- a/app.py +++ b/app.py @@ -1,135 +1,137 @@ import binascii import hashlib import json -import urllib -import os -import mimetypes import logging -from functools import wraps +import mimetypes +import os +import urllib from datetime import datetime +from functools import wraps +from typing import Any +from typing import Dict +from urllib.parse import urlencode +from urllib.parse import urlparse -import timeago import bleach import mf2py -import pymongo import piexif +import pymongo +import timeago from bson.objectid import ObjectId from flask import Flask -from flask import abort -from flask import request -from flask import redirect from flask import Response -from flask import render_template -from flask import session +from flask import abort from flask import jsonify as flask_jsonify +from flask import redirect +from flask import render_template +from flask import request +from flask import session from flask import url_for +from flask_wtf.csrf import CSRFProtect from html2text import html2text -from itsdangerous import JSONWebSignatureSerializer from itsdangerous import BadSignature +from itsdangerous import JSONWebSignatureSerializer from passlib.hash import bcrypt from u2flib_server import u2f -from urllib.parse import urlparse, urlencode from werkzeug.utils import secure_filename -from flask_wtf.csrf import CSRFProtect import activitypub import config -from activitypub import ActivityType -from activitypub import clean_activity from activitypub import embed_collection -from utils.content_helper import parse_markdown -from config import KEY -from config import DB -from config import ME -from config import ID -from config import DOMAIN -from config import USERNAME -from config import BASE_URL from config import ACTOR_SERVICE +from config import ADMIN_API_KEY +from config import BASE_URL +from config import DB +from config import DEBUG_MODE +from config import DOMAIN +from config import HEADERS +from config import ID +from config import JWT +from config import KEY +from config import ME from config import OBJECT_SERVICE from config import PASS -from config import HEADERS +from config import USERNAME from config import VERSION -from config import DEBUG_MODE -from config import JWT -from config import ADMIN_API_KEY from config import _drop_db from config import custom_cache_purge_hook -from utils.httpsig import HTTPSigAuth, verify_request -from utils.key import get_secret_key -from utils.webfinger import get_remote_follow_template -from utils.webfinger import get_actor_url -from utils.errors import Error -from utils.errors import UnexpectedActivityTypeError -from utils.errors import BadActivityError -from utils.errors import NotFromOutboxError +from little_boxes.activitypub import ActivityType +from little_boxes.activitypub import clean_activity +from little_boxes.errors import BadActivityError +from little_boxes.errors import Error +from little_boxes.errors import UnexpectedActivityTypeError +from little_boxes.httpsig import HTTPSigAuth +from little_boxes.httpsig import verify_request +from little_boxes.webfinger import get_actor_url +from little_boxes.webfinger import get_remote_follow_template +from utils.content_helper import parse_markdown from utils.errors import ActivityNotFoundError - - -from typing import Dict, Any +from utils.errors import NotFromOutboxError +from utils.key import get_secret_key app = Flask(__name__) -app.secret_key = get_secret_key('flask') -app.config.update( - WTF_CSRF_CHECK_DEFAULT=False, -) +app.secret_key = get_secret_key("flask") +app.config.update(WTF_CSRF_CHECK_DEFAULT=False) csrf = CSRFProtect(app) logger = logging.getLogger(__name__) # Hook up Flask logging with gunicorn root_logger = logging.getLogger() -if os.getenv('FLASK_DEBUG'): +if os.getenv("FLASK_DEBUG"): logger.setLevel(logging.DEBUG) root_logger.setLevel(logging.DEBUG) else: - gunicorn_logger = logging.getLogger('gunicorn.error') + gunicorn_logger = logging.getLogger("gunicorn.error") root_logger.handlers = gunicorn_logger.handlers root_logger.setLevel(gunicorn_logger.level) -SIG_AUTH = HTTPSigAuth(ID+'#main-key', KEY.privkey) +SIG_AUTH = HTTPSigAuth(KEY) def verify_pass(pwd): - return bcrypt.verify(pwd, PASS) + return bcrypt.verify(pwd, PASS) + @app.context_processor def inject_config(): - return dict( - microblogpub_version=VERSION, - config=config, - logged_in=session.get('logged_in', False), - ) + return dict( + microblogpub_version=VERSION, + config=config, + logged_in=session.get("logged_in", False), + ) + @app.after_request def set_x_powered_by(response): - response.headers['X-Powered-By'] = 'microblog.pub' + response.headers["X-Powered-By"] = "microblog.pub" return response + # HTML/templates helper ALLOWED_TAGS = [ - 'a', - 'abbr', - 'acronym', - 'b', - 'blockquote', - 'code', - 'pre', - 'em', - 'i', - 'li', - 'ol', - 'strong', - 'ul', - 'span', - 'div', - 'p', - 'h1', - 'h2', - 'h3', - 'h4', - 'h5', - 'h6', + "a", + "abbr", + "acronym", + "b", + "blockquote", + "code", + "pre", + "em", + "i", + "li", + "ol", + "strong", + "ul", + "span", + "div", + "p", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", ] @@ -166,13 +168,16 @@ def domain(url): def get_actor(url): if not url: return None - print(f'GET_ACTOR {url}') + print(f"GET_ACTOR {url}") return ACTOR_SERVICE.get(url) + @app.template_filter() def format_time(val): if val: - return datetime.strftime(datetime.strptime(val, '%Y-%m-%dT%H:%M:%SZ'), '%B %d, %Y, %H:%M %p') + return datetime.strftime( + datetime.strptime(val, "%Y-%m-%dT%H:%M:%SZ"), "%B %d, %Y, %H:%M %p" + ) return val @@ -180,26 +185,38 @@ def format_time(val): def format_timeago(val): if val: try: - return timeago.format(datetime.strptime(val, '%Y-%m-%dT%H:%M:%SZ'), datetime.utcnow()) - except: - return timeago.format(datetime.strptime(val, '%Y-%m-%dT%H:%M:%S.%fZ'), datetime.utcnow()) + return timeago.format( + datetime.strptime(val, "%Y-%m-%dT%H:%M:%SZ"), datetime.utcnow() + ) + except Exception: + return timeago.format( + datetime.strptime(val, "%Y-%m-%dT%H:%M:%S.%fZ"), datetime.utcnow() + ) return val + def _is_img(filename): filename = filename.lower() - if (filename.endswith('.png') or filename.endswith('.jpg') or filename.endswith('.jpeg') or - filename.endswith('.gif') or filename.endswith('.svg')): + if ( + filename.endswith(".png") + or filename.endswith(".jpg") + or filename.endswith(".jpeg") + or filename.endswith(".gif") + or filename.endswith(".svg") + ): return True return False + @app.template_filter() def not_only_imgs(attachment): for a in attachment: - if not _is_img(a['url']): + if not _is_img(a["url"]): return True return False + @app.template_filter() def is_img(filename): return _is_img(filename) @@ -208,28 +225,29 @@ def is_img(filename): def login_required(f): @wraps(f) def decorated_function(*args, **kwargs): - if not session.get('logged_in'): - return redirect(url_for('login', next=request.url)) + if not session.get("logged_in"): + return redirect(url_for("login", next=request.url)) return f(*args, **kwargs) + return decorated_function def _api_required(): - if session.get('logged_in'): - if request.method not in ['GET', 'HEAD']: + if session.get("logged_in"): + if request.method not in ["GET", "HEAD"]: # If a standard API request is made with a "login session", it must havw a CSRF token csrf.protect() return # Token verification - token = request.headers.get('Authorization', '').replace('Bearer ', '') + token = request.headers.get("Authorization", "").replace("Bearer ", "") if not token: # IndieAuth token - token = request.form.get('access_token', '') + token = request.form.get("access_token", "") # Will raise a BadSignature on bad auth payload = JWT.loads(token) - logger.info(f'api call by {payload}') + logger.info(f"api call by {payload}") def api_required(f): @@ -241,31 +259,36 @@ def api_required(f): abort(401) return f(*args, **kwargs) + return decorated_function def jsonify(**data): - if '@context' not in data: - data['@context'] = config.CTX_AS + if "@context" not in data: + data["@context"] = config.CTX_AS return Response( response=json.dumps(data), - headers={'Content-Type': 'application/json' if app.debug else 'application/activity+json'}, + headers={ + "Content-Type": "application/json" + if app.debug + else "application/activity+json" + }, ) def is_api_request(): - h = request.headers.get('Accept') + h = request.headers.get("Accept") if h is None: return False - h = h.split(',')[0] - if h in HEADERS or h == 'application/json': + h = h.split(",")[0] + if h in HEADERS or h == "application/json": return True return False @app.errorhandler(ValueError) def handle_value_error(error): - logger.error(f'caught value error: {error!r}') + logger.error(f"caught value error: {error!r}") response = flask_jsonify(message=error.args[0]) response.status_code = 400 return response @@ -273,7 +296,7 @@ def handle_value_error(error): @app.errorhandler(Error) def handle_activitypub_error(error): - logger.error(f'caught activitypub error {error!r}') + logger.error(f"caught activitypub error {error!r}") response = flask_jsonify(error.to_dict()) response.status_code = error.status_code return response @@ -284,99 +307,102 @@ def handle_activitypub_error(error): ####### # Login -@app.route('/logout') + +@app.route("/logout") @login_required def logout(): - session['logged_in'] = False - return redirect('/') + session["logged_in"] = False + return redirect("/") -@app.route('/login', methods=['POST', 'GET']) +@app.route("/login", methods=["POST", "GET"]) def login(): - devices = [doc['device'] for doc in DB.u2f.find()] + devices = [doc["device"] for doc in DB.u2f.find()] u2f_enabled = True if devices else False - if request.method == 'POST': + if request.method == "POST": csrf.protect() - pwd = request.form.get('pass') + pwd = request.form.get("pass") if pwd and verify_pass(pwd): if devices: - resp = json.loads(request.form.get('resp')) + resp = json.loads(request.form.get("resp")) print(resp) try: - u2f.complete_authentication(session['challenge'], resp) + u2f.complete_authentication(session["challenge"], resp) except ValueError as exc: - print('failed', exc) + print("failed", exc) abort(401) return finally: - session['challenge'] = None + session["challenge"] = None - session['logged_in'] = True - return redirect(request.args.get('redirect') or '/admin') + session["logged_in"] = True + return redirect(request.args.get("redirect") or "/admin") else: abort(401) payload = None if devices: payload = u2f.begin_authentication(ID, devices) - session['challenge'] = payload + session["challenge"] = payload return render_template( - 'login.html', - u2f_enabled=u2f_enabled, - me=ME, - payload=payload, + "login.html", u2f_enabled=u2f_enabled, me=ME, payload=payload ) -@app.route('/remote_follow', methods=['GET', 'POST']) +@app.route("/remote_follow", methods=["GET", "POST"]) def remote_follow(): - if request.method == 'GET': - return render_template('remote_follow.html') + if request.method == "GET": + return render_template("remote_follow.html") csrf.protect() - return redirect(get_remote_follow_template('@'+request.form.get('profile')).format(uri=f'{USERNAME}@{DOMAIN}')) + return redirect( + get_remote_follow_template("@" + request.form.get("profile")).format( + uri=f"{USERNAME}@{DOMAIN}" + ) + ) -@app.route('/authorize_follow', methods=['GET', 'POST']) +@app.route("/authorize_follow", methods=["GET", "POST"]) @login_required def authorize_follow(): - if request.method == 'GET': - return render_template('authorize_remote_follow.html', profile=request.args.get('profile')) + if request.method == "GET": + return render_template( + "authorize_remote_follow.html", profile=request.args.get("profile") + ) - actor = get_actor_url(request.form.get('profile')) + actor = get_actor_url(request.form.get("profile")) if not actor: abort(500) - if DB.following.find({'remote_actor': actor}).count() > 0: - return redirect('/following') + if DB.following.find({"remote_actor": actor}).count() > 0: + return redirect("/following") follow = activitypub.Follow(object=actor) follow.post_to_outbox() - return redirect('/following') + return redirect("/following") -@app.route('/u2f/register', methods=['GET', 'POST']) +@app.route("/u2f/register", methods=["GET", "POST"]) @login_required def u2f_register(): # TODO(tsileo): ensure no duplicates - if request.method == 'GET': + if request.method == "GET": payload = u2f.begin_registration(ID) - session['challenge'] = payload - return render_template( - 'u2f.html', - payload=payload, - ) + session["challenge"] = payload + return render_template("u2f.html", payload=payload) else: - resp = json.loads(request.form.get('resp')) - device, device_cert = u2f.complete_registration(session['challenge'], resp) - session['challenge'] = None - DB.u2f.insert_one({'device': device, 'cert': device_cert}) - return '' + resp = json.loads(request.form.get("resp")) + device, device_cert = u2f.complete_registration(session["challenge"], resp) + session["challenge"] = None + DB.u2f.insert_one({"device": device, "cert": device_cert}) + return "" + ####### # Activity pub routes -@app.route('/') + +@app.route("/") def index(): if is_api_request(): return jsonify(**ME) @@ -384,31 +410,41 @@ def index(): # FIXME(tsileo): implements pagination, also for the followers/following page limit = 50 q = { - 'type': 'Create', - 'activity.object.type': 'Note', - 'activity.object.inReplyTo': None, - 'meta.deleted': False, + "type": "Create", + "activity.object.type": "Note", + "activity.object.inReplyTo": None, + "meta.deleted": False, } - c = request.args.get('cursor') + c = request.args.get("cursor") if c: - q['_id'] = {'$lt': ObjectId(c)} + q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list(DB.outbox.find({'$or': [q, {'type': 'Announce', 'meta.undo': False}]}, limit=limit).sort('_id', -1)) + outbox_data = list( + DB.outbox.find( + {"$or": [q, {"type": "Announce", "meta.undo": False}]}, limit=limit + ).sort("_id", -1) + ) cursor = None if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]['_id']) + cursor = str(outbox_data[-1]["_id"]) for data in outbox_data: - if data['type'] == 'Announce': + if data["type"] == "Announce": print(data) - if data['activity']['object'].startswith('http'): - data['ref'] = {'activity': {'object': OBJECT_SERVICE.get(data['activity']['object'])}, 'meta': {}} - + if data["activity"]["object"].startswith("http"): + data["ref"] = { + "activity": { + "object": OBJECT_SERVICE.get(data["activity"]["object"]) + }, + "meta": {}, + } return render_template( - 'index.html', + "index.html", me=ME, - notes=DB.inbox.find({'type': 'Create', 'activity.object.type': 'Note', 'meta.deleted': False}).count(), + notes=DB.inbox.find( + {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} + ).count(), followers=DB.followers.count(), following=DB.following.count(), outbox_data=outbox_data, @@ -416,34 +452,40 @@ def index(): ) -@app.route('/with_replies') +@app.route("/with_replies") def with_replies(): limit = 50 - q = { - 'type': 'Create', - 'activity.object.type': 'Note', - 'meta.deleted': False, - } - c = request.args.get('cursor') + q = {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} + c = request.args.get("cursor") if c: - q['_id'] = {'$lt': ObjectId(c)} + q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list(DB.outbox.find({'$or': [q, {'type': 'Announce', 'meta.undo': False}]}, limit=limit).sort('_id', -1)) + outbox_data = list( + DB.outbox.find( + {"$or": [q, {"type": "Announce", "meta.undo": False}]}, limit=limit + ).sort("_id", -1) + ) cursor = None if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]['_id']) + cursor = str(outbox_data[-1]["_id"]) for data in outbox_data: - if data['type'] == 'Announce': + if data["type"] == "Announce": print(data) - if data['activity']['object'].startswith('http'): - data['ref'] = {'activity': {'object': OBJECT_SERVICE.get(data['activity']['object'])}, 'meta': {}} - + if data["activity"]["object"].startswith("http"): + data["ref"] = { + "activity": { + "object": OBJECT_SERVICE.get(data["activity"]["object"]) + }, + "meta": {}, + } return render_template( - 'index.html', + "index.html", me=ME, - notes=DB.inbox.find({'type': 'Create', 'activity.object.type': 'Note', 'meta.deleted': False}).count(), + notes=DB.inbox.find( + {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} + ).count(), followers=DB.followers.count(), following=DB.following.count(), outbox_data=outbox_data, @@ -452,17 +494,17 @@ def with_replies(): def _build_thread(data, include_children=True): - data['_requested'] = True - root_id = data['meta'].get('thread_root_parent', data['activity']['object']['id']) + data["_requested"] = True + root_id = data["meta"].get("thread_root_parent", data["activity"]["object"]["id"]) - thread_ids = data['meta'].get('thread_parents', []) + thread_ids = data["meta"].get("thread_parents", []) if include_children: - thread_ids.extend(data['meta'].get('thread_children', [])) + thread_ids.extend(data["meta"].get("thread_children", [])) query = { - 'activity.object.id': {'$in': thread_ids}, - 'type': 'Create', - 'meta.deleted': False, # TODO(tsileo): handle Tombstone instead of filtering them + "activity.object.id": {"$in": thread_ids}, + "type": "Create", + "meta.deleted": False, # TODO(tsileo): handle Tombstone instead of filtering them } # Fetch the root replies, and the children replies = [data] + list(DB.inbox.find(query)) + list(DB.outbox.find(query)) @@ -470,131 +512,166 @@ def _build_thread(data, include_children=True): # Index all the IDs in order to build a tree idx = {} for rep in replies: - rep_id = rep['activity']['object']['id'] + rep_id = rep["activity"]["object"]["id"] idx[rep_id] = rep.copy() - idx[rep_id]['_nodes'] = [] + idx[rep_id]["_nodes"] = [] # Build the tree for rep in replies: - rep_id = rep['activity']['object']['id'] + rep_id = rep["activity"]["object"]["id"] if rep_id == root_id: continue - reply_of = rep['activity']['object']['inReplyTo'] - idx[reply_of]['_nodes'].append(rep) + reply_of = rep["activity"]["object"]["inReplyTo"] + idx[reply_of]["_nodes"].append(rep) # Flatten the tree thread = [] + def _flatten(node, level=0): - node['_level'] = level + node["_level"] = level thread.append(node) - for snode in sorted(idx[node['activity']['object']['id']]['_nodes'], key=lambda d: d['activity']['object']['published']): - _flatten(snode, level=level+1) + for snode in sorted( + idx[node["activity"]["object"]["id"]]["_nodes"], + key=lambda d: d["activity"]["object"]["published"], + ): + _flatten(snode, level=level + 1) + _flatten(idx[root_id]) return thread -@app.route('/note/') +@app.route("/note/") def note_by_id(note_id): - data = DB.outbox.find_one({'id': note_id}) + data = DB.outbox.find_one({"id": note_id}) if not data: abort(404) - if data['meta'].get('deleted', False): + if data["meta"].get("deleted", False): abort(410) thread = _build_thread(data) + likes = list( + DB.inbox.find( + { + "meta.undo": False, + "type": ActivityType.LIKE.value, + "$or": [ + {"activity.object.id": data["activity"]["object"]["id"]}, + {"activity.object": data["activity"]["object"]["id"]}, + ], + } + ) + ) + likes = [ACTOR_SERVICE.get(doc["activity"]["actor"]) for doc in likes] - likes = list(DB.inbox.find({ - 'meta.undo': False, - 'type': ActivityType.LIKE.value, - '$or': [{'activity.object.id': data['activity']['object']['id']}, - {'activity.object': data['activity']['object']['id']}], - })) - likes = [ACTOR_SERVICE.get(doc['activity']['actor']) for doc in likes] + shares = list( + DB.inbox.find( + { + "meta.undo": False, + "type": ActivityType.ANNOUNCE.value, + "$or": [ + {"activity.object.id": data["activity"]["object"]["id"]}, + {"activity.object": data["activity"]["object"]["id"]}, + ], + } + ) + ) + shares = [ACTOR_SERVICE.get(doc["activity"]["actor"]) for doc in shares] - shares = list(DB.inbox.find({ - 'meta.undo': False, - 'type': ActivityType.ANNOUNCE.value, - '$or': [{'activity.object.id': data['activity']['object']['id']}, - {'activity.object': data['activity']['object']['id']}], - })) - shares = [ACTOR_SERVICE.get(doc['activity']['actor']) for doc in shares] - - return render_template('note.html', likes=likes, shares=shares, me=ME, thread=thread, note=data) - - -@app.route('/nodeinfo') -def nodeinfo(): - return Response( - headers={'Content-Type': 'application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#'}, - response=json.dumps({ - 'version': '2.0', - 'software': {'name': 'microblogpub', 'version': f'Microblog.pub {VERSION}'}, - 'protocols': ['activitypub'], - 'services': {'inbound': [], 'outbound': []}, - 'openRegistrations': False, - 'usage': {'users': {'total': 1}, 'localPosts': DB.outbox.count()}, - 'metadata': { - 'sourceCode': 'https://github.com/tsileo/microblog.pub', - 'nodeName': f'@{USERNAME}@{DOMAIN}', - }, - }), + return render_template( + "note.html", likes=likes, shares=shares, me=ME, thread=thread, note=data ) -@app.route('/.well-known/nodeinfo') +@app.route("/nodeinfo") +def nodeinfo(): + return Response( + headers={ + "Content-Type": "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#" + }, + response=json.dumps( + { + "version": "2.0", + "software": { + "name": "microblogpub", + "version": f"Microblog.pub {VERSION}", + }, + "protocols": ["activitypub"], + "services": {"inbound": [], "outbound": []}, + "openRegistrations": False, + "usage": {"users": {"total": 1}, "localPosts": DB.outbox.count()}, + "metadata": { + "sourceCode": "https://github.com/tsileo/microblog.pub", + "nodeName": f"@{USERNAME}@{DOMAIN}", + }, + } + ), + ) + + +@app.route("/.well-known/nodeinfo") def wellknown_nodeinfo(): return flask_jsonify( links=[ { - 'rel': 'http://nodeinfo.diaspora.software/ns/schema/2.0', - 'href': f'{ID}/nodeinfo', + "rel": "http://nodeinfo.diaspora.software/ns/schema/2.0", + "href": f"{ID}/nodeinfo", } - - ], + ] ) -@app.route('/.well-known/webfinger') +@app.route("/.well-known/webfinger") def wellknown_webfinger(): """Enable WebFinger support, required for Mastodon interopability.""" - resource = request.args.get('resource') - if resource not in [f'acct:{USERNAME}@{DOMAIN}', ID]: + resource = request.args.get("resource") + if resource not in [f"acct:{USERNAME}@{DOMAIN}", ID]: abort(404) out = { - "subject": f'acct:{USERNAME}@{DOMAIN}', + "subject": f"acct:{USERNAME}@{DOMAIN}", "aliases": [ID], "links": [ - {"rel": "http://webfinger.net/rel/profile-page", "type": "text/html", "href": BASE_URL}, + { + "rel": "http://webfinger.net/rel/profile-page", + "type": "text/html", + "href": BASE_URL, + }, {"rel": "self", "type": "application/activity+json", "href": ID}, - {"rel":"http://ostatus.org/schema/1.0/subscribe", "template": BASE_URL+"/authorize_follow?profile={uri}"}, + { + "rel": "http://ostatus.org/schema/1.0/subscribe", + "template": BASE_URL + "/authorize_follow?profile={uri}", + }, ], } return Response( response=json.dumps(out), - headers={'Content-Type': 'application/jrd+json; charset=utf-8' if not app.debug else 'application/json'}, + headers={ + "Content-Type": "application/jrd+json; charset=utf-8" + if not app.debug + else "application/json" + }, ) def add_extra_collection(raw_doc: Dict[str, Any]) -> Dict[str, Any]: - if raw_doc['activity']['type'] != ActivityType.CREATE.value: + if raw_doc["activity"]["type"] != ActivityType.CREATE.value: return raw_doc - raw_doc['activity']['object']['replies'] = embed_collection( - raw_doc.get('meta', {}).get('count_direct_reply', 0), + raw_doc["activity"]["object"]["replies"] = embed_collection( + raw_doc.get("meta", {}).get("count_direct_reply", 0), f'{ID}/outbox/{raw_doc["id"]}/replies', ) - raw_doc['activity']['object']['likes'] = embed_collection( - raw_doc.get('meta', {}).get('count_like', 0), + raw_doc["activity"]["object"]["likes"] = embed_collection( + raw_doc.get("meta", {}).get("count_like", 0), f'{ID}/outbox/{raw_doc["id"]}/likes', ) - raw_doc['activity']['object']['shares'] = embed_collection( - raw_doc.get('meta', {}).get('count_boost', 0), + raw_doc["activity"]["object"]["shares"] = embed_collection( + raw_doc.get("meta", {}).get("count_boost", 0), f'{ID}/outbox/{raw_doc["id"]}/shares', ) @@ -602,37 +679,38 @@ def add_extra_collection(raw_doc: Dict[str, Any]) -> Dict[str, Any]: def remove_context(activity: Dict[str, Any]) -> Dict[str, Any]: - if '@context' in activity: - del activity['@context'] + if "@context" in activity: + del activity["@context"] return activity def activity_from_doc(raw_doc: Dict[str, Any], embed: bool = False) -> Dict[str, Any]: raw_doc = add_extra_collection(raw_doc) - activity = clean_activity(raw_doc['activity']) + activity = clean_activity(raw_doc["activity"]) if embed: return remove_context(activity) return activity - -@app.route('/outbox', methods=['GET', 'POST']) +@app.route("/outbox", methods=["GET", "POST"]) def outbox(): - if request.method == 'GET': + if request.method == "GET": if not is_api_request(): abort(404) # TODO(tsileo): filter the outbox if not authenticated # FIXME(tsileo): filter deleted, add query support for build_ordered_collection q = { - 'meta.deleted': False, - #'type': {'$in': [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, + "meta.deleted": False, + # 'type': {'$in': [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, } - return jsonify(**activitypub.build_ordered_collection( - DB.outbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: activity_from_doc(doc, embed=True), - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.outbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: activity_from_doc(doc, embed=True), + ) + ) # Handle POST request try: @@ -652,203 +730,207 @@ def outbox(): # Purge the cache if a custom hook is set, as new content was published custom_cache_purge_hook() - return Response(status=201, headers={'Location': activity.id}) + return Response(status=201, headers={"Location": activity.id}) -@app.route('/outbox/') +@app.route("/outbox/") def outbox_detail(item_id): - doc = DB.outbox.find_one({'id': item_id}) - if doc['meta'].get('deleted', False): - obj = activitypub.parse_activity(doc['activity']) + doc = DB.outbox.find_one({"id": item_id}) + if doc["meta"].get("deleted", False): + obj = activitypub.parse_activity(doc["activity"]) resp = jsonify(**obj.get_object().get_tombstone()) resp.status_code = 410 return resp return jsonify(**activity_from_doc(doc)) -@app.route('/outbox//activity') +@app.route("/outbox//activity") def outbox_activity(item_id): # TODO(tsileo): handle Tombstone - data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + data = DB.outbox.find_one({"id": item_id, "meta.deleted": False}) if not data: abort(404) obj = activity_from_doc(data) - if obj['type'] != ActivityType.CREATE.value: + if obj["type"] != ActivityType.CREATE.value: abort(404) - return jsonify(**obj['object']) + return jsonify(**obj["object"]) -@app.route('/outbox//replies') +@app.route("/outbox//replies") def outbox_activity_replies(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + data = DB.outbox.find_one({"id": item_id, "meta.deleted": False}) if not data: abort(404) - obj = activitypub.parse_activity(data['activity']) + obj = activitypub.parse_activity(data["activity"]) if obj.type_enum != ActivityType.CREATE: abort(404) q = { - 'meta.deleted': False, - 'type': ActivityType.CREATE.value, - 'activity.object.inReplyTo': obj.get_object().id, + "meta.deleted": False, + "type": ActivityType.CREATE.value, + "activity.object.inReplyTo": obj.get_object().id, } - return jsonify(**activitypub.build_ordered_collection( - DB.inbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: doc['activity']['object'], - col_name=f'outbox/{item_id}/replies', - first_page=request.args.get('page') == 'first', - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.inbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: doc["activity"]["object"], + col_name=f"outbox/{item_id}/replies", + first_page=request.args.get("page") == "first", + ) + ) -@app.route('/outbox//likes') +@app.route("/outbox//likes") def outbox_activity_likes(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + data = DB.outbox.find_one({"id": item_id, "meta.deleted": False}) if not data: abort(404) - obj = activitypub.parse_activity(data['activity']) + obj = activitypub.parse_activity(data["activity"]) if obj.type_enum != ActivityType.CREATE: abort(404) q = { - 'meta.undo': False, - 'type': ActivityType.LIKE.value, - '$or': [{'activity.object.id': obj.get_object().id}, - {'activity.object': obj.get_object().id}], + "meta.undo": False, + "type": ActivityType.LIKE.value, + "$or": [ + {"activity.object.id": obj.get_object().id}, + {"activity.object": obj.get_object().id}, + ], } - return jsonify(**activitypub.build_ordered_collection( - DB.inbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: remove_context(doc['activity']), - col_name=f'outbox/{item_id}/likes', - first_page=request.args.get('page') == 'first', - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.inbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: remove_context(doc["activity"]), + col_name=f"outbox/{item_id}/likes", + first_page=request.args.get("page") == "first", + ) + ) -@app.route('/outbox//shares') +@app.route("/outbox//shares") def outbox_activity_shares(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + data = DB.outbox.find_one({"id": item_id, "meta.deleted": False}) if not data: abort(404) - obj = activitypub.parse_activity(data['activity']) + obj = activitypub.parse_activity(data["activity"]) if obj.type_enum != ActivityType.CREATE: abort(404) q = { - 'meta.undo': False, - 'type': ActivityType.ANNOUNCE.value, - '$or': [{'activity.object.id': obj.get_object().id}, - {'activity.object': obj.get_object().id}], + "meta.undo": False, + "type": ActivityType.ANNOUNCE.value, + "$or": [ + {"activity.object.id": obj.get_object().id}, + {"activity.object": obj.get_object().id}, + ], } - return jsonify(**activitypub.build_ordered_collection( - DB.inbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: remove_context(doc['activity']), - col_name=f'outbox/{item_id}/shares', - first_page=request.args.get('page') == 'first', - )) - - -@app.route('/admin', methods=['GET']) -@login_required -def admin(): - q = { - 'meta.deleted': False, - 'meta.undo': False, - 'type': ActivityType.LIKE.value, - } - col_liked = DB.outbox.count(q) - - return render_template( - 'admin.html', - instances=list(DB.instances.find()), - inbox_size=DB.inbox.count(), - outbox_size=DB.outbox.count(), - object_cache_size=DB.objects_cache.count(), - actor_cache_size=DB.actors_cache.count(), - col_liked=col_liked, - col_followers=DB.followers.count(), - col_following=DB.following.count(), + return jsonify( + **activitypub.build_ordered_collection( + DB.inbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: remove_context(doc["activity"]), + col_name=f"outbox/{item_id}/shares", + first_page=request.args.get("page") == "first", + ) ) -@app.route('/new', methods=['GET']) +@app.route("/admin", methods=["GET"]) +@login_required +def admin(): + q = {"meta.deleted": False, "meta.undo": False, "type": ActivityType.LIKE.value} + col_liked = DB.outbox.count(q) + + return render_template( + "admin.html", + instances=list(DB.instances.find()), + inbox_size=DB.inbox.count(), + outbox_size=DB.outbox.count(), + object_cache_size=DB.objects_cache.count(), + actor_cache_size=DB.actors_cache.count(), + col_liked=col_liked, + col_followers=DB.followers.count(), + col_following=DB.following.count(), + ) + + +@app.route("/new", methods=["GET"]) @login_required def new(): reply_id = None - content = '' + content = "" thread = [] - if request.args.get('reply'): - data = DB.inbox.find_one({'activity.object.id': request.args.get('reply')}) + if request.args.get("reply"): + data = DB.inbox.find_one({"activity.object.id": request.args.get("reply")}) if not data: - data = DB.outbox.find_one({'activity.object.id': request.args.get('reply')}) + data = DB.outbox.find_one({"activity.object.id": request.args.get("reply")}) if not data: abort(400) - reply = activitypub.parse_activity(data['activity']) + reply = activitypub.parse_activity(data["activity"]) reply_id = reply.id if reply.type_enum == ActivityType.CREATE: reply_id = reply.get_object().id actor = reply.get_actor() domain = urlparse(actor.id).netloc # FIXME(tsileo): if reply of reply, fetch all participants - content = f'@{actor.preferredUsername}@{domain} ' - thread = _build_thread( - data, - include_children=False, - ) + content = f"@{actor.preferredUsername}@{domain} " + thread = _build_thread(data, include_children=False) - return render_template( - 'new.html', - reply=reply_id, - content=content, - thread=thread, - ) + return render_template("new.html", reply=reply_id, content=content, thread=thread) -@app.route('/notifications') +@app.route("/notifications") @login_required def notifications(): # FIXME(tsileo): implements pagination, also for the followers/following page limit = 50 q = { - 'type': 'Create', - 'activity.object.tag.type': 'Mention', - 'activity.object.tag.name': f'@{USERNAME}@{DOMAIN}', - 'meta.deleted': False, + "type": "Create", + "activity.object.tag.type": "Mention", + "activity.object.tag.name": f"@{USERNAME}@{DOMAIN}", + "meta.deleted": False, } # TODO(tsileo): also include replies via regex on Create replyTo - q = {'$or': [q, {'type': 'Follow'}, {'type': 'Accept'}, {'type': 'Undo', 'activity.object.type': 'Follow'}, - {'type': 'Announce', 'activity.object': {'$regex': f'^{BASE_URL}'}}, - {'type': 'Create', 'activity.object.inReplyTo': {'$regex': f'^{BASE_URL}'}}, - ]} + q = { + "$or": [ + q, + {"type": "Follow"}, + {"type": "Accept"}, + {"type": "Undo", "activity.object.type": "Follow"}, + {"type": "Announce", "activity.object": {"$regex": f"^{BASE_URL}"}}, + {"type": "Create", "activity.object.inReplyTo": {"$regex": f"^{BASE_URL}"}}, + ] + } print(q) - c = request.args.get('cursor') + c = request.args.get("cursor") if c: - q['_id'] = {'$lt': ObjectId(c)} + q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list(DB.inbox.find(q, limit=limit).sort('_id', -1)) + outbox_data = list(DB.inbox.find(q, limit=limit).sort("_id", -1)) cursor = None if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]['_id']) + cursor = str(outbox_data[-1]["_id"]) # TODO(tsileo): fix the annonce handling, copy it from /stream - #for data in outbox_data: + # for data in outbox_data: # if data['type'] == 'Announce': # print(data) # if data['activity']['object'].startswith('http') and data['activity']['object'] in objcache: @@ -857,14 +939,10 @@ def notifications(): # else: # out.append(data) - return render_template( - 'stream.html', - inbox_data=outbox_data, - cursor=cursor, - ) + return render_template("stream.html", inbox_data=outbox_data, cursor=cursor) -@app.route('/api/key') +@app.route("/api/key") @login_required def api_user_key(): return flask_jsonify(api_key=ADMIN_API_KEY) @@ -878,25 +956,29 @@ def _user_api_arg(key: str, **kwargs): oid = request.args.get(key) or request.form.get(key) if not oid: - if 'default' in kwargs: - return kwargs.get('default') + if "default" in kwargs: + return kwargs.get("default") - raise ValueError(f'missing {key}') + raise ValueError(f"missing {key}") return oid def _user_api_get_note(from_outbox: bool = False): - oid = _user_api_arg('id') - note = activitypub.parse_activity(OBJECT_SERVICE.get(oid), expected=ActivityType.NOTE) + oid = _user_api_arg("id") + note = activitypub.parse_activity( + OBJECT_SERVICE.get(oid), expected=ActivityType.NOTE + ) if from_outbox and not note.id.startswith(ID): - raise NotFromOutboxError(f'cannot delete {note.id}, id must be owned by the server') + raise NotFromOutboxError( + f"cannot delete {note.id}, id must be owned by the server" + ) return note def _user_api_response(**kwargs): - _redirect = _user_api_arg('redirect', default=None) + _redirect = _user_api_arg("redirect", default=None) if _redirect: return redirect(_redirect) @@ -905,7 +987,7 @@ def _user_api_response(**kwargs): return resp -@app.route('/api/note/delete', methods=['POST']) +@app.route("/api/note/delete", methods=["POST"]) @api_required def api_delete(): """API endpoint to delete a Note activity.""" @@ -917,7 +999,7 @@ def api_delete(): return _user_api_response(activity=delete.id) -@app.route('/api/boost', methods=['POST']) +@app.route("/api/boost", methods=["POST"]) @api_required def api_boost(): note = _user_api_get_note() @@ -928,7 +1010,7 @@ def api_boost(): return _user_api_response(activity=announce.id) -@app.route('/api/like', methods=['POST']) +@app.route("/api/like", methods=["POST"]) @api_required def api_like(): note = _user_api_get_note() @@ -939,15 +1021,15 @@ def api_like(): return _user_api_response(activity=like.id) -@app.route('/api/undo', methods=['POST']) +@app.route("/api/undo", methods=["POST"]) @api_required def api_undo(): - oid = _user_api_arg('id') - doc = DB.outbox.find_one({'$or': [{'id': oid}, {'remote_id': oid}]}) + oid = _user_api_arg("id") + doc = DB.outbox.find_one({"$or": [{"id": oid}, {"remote_id": oid}]}) if not doc: - raise ActivityNotFoundError(f'cannot found {oid}') + raise ActivityNotFoundError(f"cannot found {oid}") - obj = activitypub.parse_activity(doc.get('activity')) + obj = activitypub.parse_activity(doc.get("activity")) # FIXME(tsileo): detect already undo-ed and make this API call idempotent undo = obj.build_undo() undo.post_to_outbox() @@ -955,58 +1037,60 @@ def api_undo(): return _user_api_response(activity=undo.id) -@app.route('/stream') +@app.route("/stream") @login_required def stream(): # FIXME(tsileo): implements pagination, also for the followers/following page limit = 100 q = { - 'type': 'Create', - 'activity.object.type': 'Note', - 'activity.object.inReplyTo': None, - 'meta.deleted': False, + "type": "Create", + "activity.object.type": "Note", + "activity.object.inReplyTo": None, + "meta.deleted": False, } - c = request.args.get('cursor') + c = request.args.get("cursor") if c: - q['_id'] = {'$lt': ObjectId(c)} + q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list(DB.inbox.find( - { - '$or': [ - q, - { - 'type': 'Announce', - }, - ] - }, limit=limit).sort('activity.published', -1)) + outbox_data = list( + DB.inbox.find({"$or": [q, {"type": "Announce"}]}, limit=limit).sort( + "activity.published", -1 + ) + ) cursor = None if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]['_id']) + cursor = str(outbox_data[-1]["_id"]) out = [] objcache = {} - cached = list(DB.objects_cache.find({'meta.part_of_stream': True}, limit=limit*3).sort('meta.announce_published', -1)) + cached = list( + DB.objects_cache.find({"meta.part_of_stream": True}, limit=limit * 3).sort( + "meta.announce_published", -1 + ) + ) for c in cached: - objcache[c['object_id']] = c['cached_object'] + objcache[c["object_id"]] = c["cached_object"] for data in outbox_data: - if data['type'] == 'Announce': - if data['activity']['object'].startswith('http') and data['activity']['object'] in objcache: - data['ref'] = {'activity': {'object': objcache[data['activity']['object']]}, 'meta': {}} + if data["type"] == "Announce": + if ( + data["activity"]["object"].startswith("http") + and data["activity"]["object"] in objcache + ): + data["ref"] = { + "activity": {"object": objcache[data["activity"]["object"]]}, + "meta": {}, + } out.append(data) else: - print('OMG', data) + print("OMG", data) else: out.append(data) - return render_template( - 'stream.html', - inbox_data=out, - cursor=cursor, - ) + return render_template("stream.html", inbox_data=out, cursor=cursor) -@app.route('/inbox', methods=['GET', 'POST']) +@app.route("/inbox", methods=["GET", "POST"]) def inbox(): - if request.method == 'GET': + if request.method == "GET": if not is_api_request(): abort(404) try: @@ -1014,135 +1098,136 @@ def inbox(): except BadSignature: abort(404) - return jsonify(**activitypub.build_ordered_collection( - DB.inbox, - q={'meta.deleted': False}, - cursor=request.args.get('cursor'), - map_func=lambda doc: remove_context(doc['activity']), - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.inbox, + q={"meta.deleted": False}, + cursor=request.args.get("cursor"), + map_func=lambda doc: remove_context(doc["activity"]), + ) + ) data = request.get_json(force=True) - logger.debug(f'req_headers={request.headers}') - logger.debug(f'raw_data={data}') + logger.debug(f"req_headers={request.headers}") + logger.debug(f"raw_data={data}") try: if not verify_request(ACTOR_SERVICE): - raise Exception('failed to verify request') + raise Exception("failed to verify request") except Exception: - logger.exception('failed to verify request, trying to verify the payload by fetching the remote') + logger.exception( + "failed to verify request, trying to verify the payload by fetching the remote" + ) try: - data = OBJECT_SERVICE.get(data['id']) + data = OBJECT_SERVICE.get(data["id"]) except Exception: logger.exception(f'failed to fetch remote id at {data["id"]}') return Response( status=422, - headers={'Content-Type': 'application/json'}, - response=json.dumps({'error': 'failed to verify request (using HTTP signatures or fetching the IRI)'}), + headers={"Content-Type": "application/json"}, + response=json.dumps( + { + "error": "failed to verify request (using HTTP signatures or fetching the IRI)" + } + ), ) activity = activitypub.parse_activity(data) - logger.debug(f'inbox activity={activity}/{data}') + logger.debug(f"inbox activity={activity}/{data}") activity.process_from_inbox() - return Response( - status=201, - ) + return Response(status=201) -@app.route('/api/debug', methods=['GET', 'DELETE']) +@app.route("/api/debug", methods=["GET", "DELETE"]) @api_required def api_debug(): """Endpoint used/needed for testing, only works in DEBUG_MODE.""" if not DEBUG_MODE: - return flask_jsonify(message='DEBUG_MODE is off') + return flask_jsonify(message="DEBUG_MODE is off") - if request.method == 'DELETE': + if request.method == "DELETE": _drop_db() - return flask_jsonify(message='DB dropped') + return flask_jsonify(message="DB dropped") - return flask_jsonify( - inbox=DB.inbox.count(), - outbox=DB.outbox.count(), - ) + return flask_jsonify(inbox=DB.inbox.count(), outbox=DB.outbox.count()) -@app.route('/api/upload', methods=['POST']) +@app.route("/api/upload", methods=["POST"]) @api_required def api_upload(): - file = request.files['file'] + file = request.files["file"] rfilename = secure_filename(file.filename) prefix = hashlib.sha256(os.urandom(32)).hexdigest()[:6] mtype = mimetypes.guess_type(rfilename)[0] - filename = f'{prefix}_{rfilename}' - file.save(os.path.join('static', 'media', filename)) + filename = f"{prefix}_{rfilename}" + file.save(os.path.join("static", "media", filename)) # Remove EXIF metadata - if filename.lower().endswith('.jpg') or filename.lower().endswith('.jpeg'): - piexif.remove(os.path.join('static', 'media', filename)) + if filename.lower().endswith(".jpg") or filename.lower().endswith(".jpeg"): + piexif.remove(os.path.join("static", "media", filename)) - print('upload OK') + print("upload OK") print(filename) attachment = [ - {'mediaType': mtype, - 'name': rfilename, - 'type': 'Document', - 'url': BASE_URL + f'/static/media/{filename}' - }, + { + "mediaType": mtype, + "name": rfilename, + "type": "Document", + "url": BASE_URL + f"/static/media/{filename}", + } ] print(attachment) - content = request.args.get('content') - to = request.args.get('to') + content = request.args.get("content") + to = request.args.get("to") note = activitypub.Note( - cc=[ID+'/followers'], + cc=[ID + "/followers"], to=[to if to else config.AS_PUBLIC], content=content, # TODO(tsileo): handle markdown attachment=attachment, ) - print('post_note_init') + print("post_note_init") print(note) create = note.build_create() print(create) print(create.to_dict()) create.post_to_outbox() - print('posted') + print("posted") - return Response( - status=201, - response='OK', - ) + return Response(status=201, response="OK") -@app.route('/api/new_note', methods=['POST']) +@app.route("/api/new_note", methods=["POST"]) @api_required def api_new_note(): - source = _user_api_arg('content') + source = _user_api_arg("content") if not source: - raise ValueError('missing content') + raise ValueError("missing content") _reply, reply = None, None try: - _reply = _user_api_arg('reply') + _reply = _user_api_arg("reply") except ValueError: pass content, tags = parse_markdown(source) - to = request.args.get('to') - cc = [ID+'/followers'] + to = request.args.get("to") + cc = [ID + "/followers"] if _reply: reply = activitypub.parse_activity(OBJECT_SERVICE.get(_reply)) cc.append(reply.attributedTo) for tag in tags: - if tag['type'] == 'Mention': - cc.append(tag['href']) + if tag["type"] == "Mention": + cc.append(tag["href"]) note = activitypub.Note( cc=list(set(cc)), to=[to if to else config.AS_PUBLIC], content=content, tag=tags, - source={'mediaType': 'text/markdown', 'content': source}, - inReplyTo=reply.id if reply else None + source={"mediaType": "text/markdown", "content": source}, + inReplyTo=reply.id if reply else None, ) create = note.build_create() create.post_to_outbox() @@ -1150,27 +1235,27 @@ def api_new_note(): return _user_api_response(activity=create.id) -@app.route('/api/stream') +@app.route("/api/stream") @api_required def api_stream(): return Response( - response=json.dumps(activitypub.build_inbox_json_feed('/api/stream', request.args.get('cursor'))), - headers={'Content-Type': 'application/json'}, + response=json.dumps( + activitypub.build_inbox_json_feed("/api/stream", request.args.get("cursor")) + ), + headers={"Content-Type": "application/json"}, ) -@app.route('/api/block', methods=['POST']) +@app.route("/api/block", methods=["POST"]) @api_required def api_block(): - actor = _user_api_arg('actor') + actor = _user_api_arg("actor") - existing = DB.outbox.find_one({ - 'type': ActivityType.BLOCK.value, - 'activity.object': actor, - 'meta.undo': False, - }) + existing = DB.outbox.find_one( + {"type": ActivityType.BLOCK.value, "activity.object": actor, "meta.undo": False} + ) if existing: - return _user_api_response(activity=existing['activity']['id']) + return _user_api_response(activity=existing["activity"]["id"]) block = activitypub.Block(object=actor) block.post_to_outbox() @@ -1178,14 +1263,14 @@ def api_block(): return _user_api_response(activity=block.id) -@app.route('/api/follow', methods=['POST']) +@app.route("/api/follow", methods=["POST"]) @api_required def api_follow(): - actor = _user_api_arg('actor') + actor = _user_api_arg("actor") - existing = DB.following.find_one({'remote_actor': actor}) + existing = DB.following.find_one({"remote_actor": actor}) if existing: - return _user_api_response(activity=existing['activity']['id']) + return _user_api_response(activity=existing["activity"]["id"]) follow = activitypub.Follow(object=actor) follow.post_to_outbox() @@ -1193,109 +1278,122 @@ def api_follow(): return _user_api_response(activity=follow.id) -@app.route('/followers') +@app.route("/followers") def followers(): if is_api_request(): return jsonify( **activitypub.build_ordered_collection( DB.followers, - cursor=request.args.get('cursor'), - map_func=lambda doc: doc['remote_actor'], + cursor=request.args.get("cursor"), + map_func=lambda doc: doc["remote_actor"], ) ) - followers = [ACTOR_SERVICE.get(doc['remote_actor']) for doc in DB.followers.find(limit=50)] + followers = [ + ACTOR_SERVICE.get(doc["remote_actor"]) for doc in DB.followers.find(limit=50) + ] return render_template( - 'followers.html', + "followers.html", me=ME, - notes=DB.inbox.find({'object.object.type': 'Note'}).count(), + notes=DB.inbox.find({"object.object.type": "Note"}).count(), followers=DB.followers.count(), following=DB.following.count(), followers_data=followers, ) -@app.route('/following') +@app.route("/following") def following(): if is_api_request(): return jsonify( **activitypub.build_ordered_collection( DB.following, - cursor=request.args.get('cursor'), - map_func=lambda doc: doc['remote_actor'], - ), + cursor=request.args.get("cursor"), + map_func=lambda doc: doc["remote_actor"], + ) ) - following = [ACTOR_SERVICE.get(doc['remote_actor']) for doc in DB.following.find(limit=50)] + following = [ + ACTOR_SERVICE.get(doc["remote_actor"]) for doc in DB.following.find(limit=50) + ] return render_template( - 'following.html', + "following.html", me=ME, - notes=DB.inbox.find({'object.object.type': 'Note'}).count(), + notes=DB.inbox.find({"object.object.type": "Note"}).count(), followers=DB.followers.count(), following=DB.following.count(), following_data=following, ) -@app.route('/tags/') +@app.route("/tags/") def tags(tag): - if not DB.outbox.count({'activity.object.tag.type': 'Hashtag', 'activity.object.tag.name': '#'+tag}): + if not DB.outbox.count( + {"activity.object.tag.type": "Hashtag", "activity.object.tag.name": "#" + tag} + ): abort(404) if not is_api_request(): return render_template( - 'tags.html', + "tags.html", tag=tag, - outbox_data=DB.outbox.find({'type': 'Create', 'activity.object.type': 'Note', 'meta.deleted': False, - 'activity.object.tag.type': 'Hashtag', - 'activity.object.tag.name': '#'+tag}), + outbox_data=DB.outbox.find( + { + "type": "Create", + "activity.object.type": "Note", + "meta.deleted": False, + "activity.object.tag.type": "Hashtag", + "activity.object.tag.name": "#" + tag, + } + ), ) q = { - 'meta.deleted': False, - 'meta.undo': False, - 'type': ActivityType.CREATE.value, - 'activity.object.tag.type': 'Hashtag', - 'activity.object.tag.name': '#'+tag, + "meta.deleted": False, + "meta.undo": False, + "type": ActivityType.CREATE.value, + "activity.object.tag.type": "Hashtag", + "activity.object.tag.name": "#" + tag, } - return jsonify(**activitypub.build_ordered_collection( - DB.outbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: doc['activity']['object']['id'], - col_name=f'tags/{tag}', - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.outbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: doc["activity"]["object"]["id"], + col_name=f"tags/{tag}", + ) + ) -@app.route('/liked') +@app.route("/liked") def liked(): if not is_api_request(): abort(404) - q = { - 'meta.deleted': False, - 'meta.undo': False, - 'type': ActivityType.LIKE.value, - } - return jsonify(**activitypub.build_ordered_collection( - DB.outbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: doc['activity']['object'], - col_name='liked', - )) + q = {"meta.deleted": False, "meta.undo": False, "type": ActivityType.LIKE.value} + return jsonify( + **activitypub.build_ordered_collection( + DB.outbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: doc["activity"]["object"], + col_name="liked", + ) + ) + ####### # IndieAuth def build_auth_resp(payload): - if request.headers.get('Accept') == 'application/json': + if request.headers.get("Accept") == "application/json": return Response( status=200, - headers={'Content-Type': 'application/json'}, + headers={"Content-Type": "application/json"}, response=json.dumps(payload), ) return Response( status=200, - headers={'Content-Type': 'application/x-www-form-urlencoded'}, + headers={"Content-Type": "application/x-www-form-urlencoded"}, response=urlencode(payload), ) @@ -1308,43 +1406,37 @@ def _get_prop(props, name, default=None): return items return default + def get_client_id_data(url): data = mf2py.parse(url=url) - for item in data['items']: - if 'h-x-app' in item['type'] or 'h-app' in item['type']: - props = item.get('properties', {}) + for item in data["items"]: + if "h-x-app" in item["type"] or "h-app" in item["type"]: + props = item.get("properties", {}) print(props) return dict( - logo=_get_prop(props, 'logo'), - name=_get_prop(props, 'name'), - url=_get_prop(props, 'url'), + logo=_get_prop(props, "logo"), + name=_get_prop(props, "name"), + url=_get_prop(props, "url"), ) - return dict( - logo=None, - name=url, - url=url, - ) + return dict(logo=None, name=url, url=url) -@app.route('/indieauth/flow', methods=['POST']) +@app.route("/indieauth/flow", methods=["POST"]) @login_required def indieauth_flow(): auth = dict( - scope=' '.join(request.form.getlist('scopes')), - me=request.form.get('me'), - client_id=request.form.get('client_id'), - state=request.form.get('state'), - redirect_uri=request.form.get('redirect_uri'), - response_type=request.form.get('response_type'), + scope=" ".join(request.form.getlist("scopes")), + me=request.form.get("me"), + client_id=request.form.get("client_id"), + state=request.form.get("state"), + redirect_uri=request.form.get("redirect_uri"), + response_type=request.form.get("response_type"), ) - code = binascii.hexlify(os.urandom(8)).decode('utf-8') - auth.update( - code=code, - verified=False, - ) + code = binascii.hexlify(os.urandom(8)).decode("utf-8") + auth.update(code=code, verified=False) print(auth) - if not auth['redirect_uri']: + if not auth["redirect_uri"]: abort(500) DB.indieauth.insert_one(auth) @@ -1356,21 +1448,21 @@ def indieauth_flow(): # @app.route('/indieauth', methods=['GET', 'POST']) def indieauth_endpoint(): - if request.method == 'GET': - if not session.get('logged_in'): - return redirect(url_for('login', next=request.url)) + if request.method == "GET": + if not session.get("logged_in"): + return redirect(url_for("login", next=request.url)) - me = request.args.get('me') + me = request.args.get("me") # FIXME(tsileo): ensure me == ID - client_id = request.args.get('client_id') - redirect_uri = request.args.get('redirect_uri') - state = request.args.get('state', '') - response_type = request.args.get('response_type', 'id') - scope = request.args.get('scope', '').split() + client_id = request.args.get("client_id") + redirect_uri = request.args.get("redirect_uri") + state = request.args.get("state", "") + response_type = request.args.get("response_type", "id") + scope = request.args.get("scope", "").split() - print('STATE', state) + print("STATE", state) return render_template( - 'indieauth_flow.html', + "indieauth_flow.html", client=get_client_id_data(client_id), scopes=scope, redirect_uri=redirect_uri, @@ -1381,14 +1473,18 @@ def indieauth_endpoint(): ) # Auth verification via POST - code = request.form.get('code') - redirect_uri = request.form.get('redirect_uri') - client_id = request.form.get('client_id') + code = request.form.get("code") + redirect_uri = request.form.get("redirect_uri") + client_id = request.form.get("client_id") auth = DB.indieauth.find_one_and_update( - {'code': code, 'redirect_uri': redirect_uri, 'client_id': client_id}, #}, # , 'verified': False}, - {'$set': {'verified': True}}, - sort=[('_id', pymongo.DESCENDING)], + { + "code": code, + "redirect_uri": redirect_uri, + "client_id": client_id, + }, # }, # , 'verified': False}, + {"$set": {"verified": True}}, + sort=[("_id", pymongo.DESCENDING)], ) print(auth) print(code, redirect_uri, client_id) @@ -1397,33 +1493,42 @@ def indieauth_endpoint(): abort(403) return - session['logged_in'] = True - me = auth['me'] - state = auth['state'] - scope = ' '.join(auth['scope']) - print('STATE', state) - return build_auth_resp({'me': me, 'state': state, 'scope': scope}) + session["logged_in"] = True + me = auth["me"] + state = auth["state"] + scope = " ".join(auth["scope"]) + print("STATE", state) + return build_auth_resp({"me": me, "state": state, "scope": scope}) -@app.route('/token', methods=['GET', 'POST']) +@app.route("/token", methods=["GET", "POST"]) def token_endpoint(): - if request.method == 'POST': - code = request.form.get('code') - me = request.form.get('me') - redirect_uri = request.form.get('redirect_uri') - client_id = request.form.get('client_id') + if request.method == "POST": + code = request.form.get("code") + me = request.form.get("me") + redirect_uri = request.form.get("redirect_uri") + client_id = request.form.get("client_id") - auth = DB.indieauth.find_one({'code': code, 'me': me, 'redirect_uri': redirect_uri, 'client_id': client_id}) + auth = DB.indieauth.find_one( + { + "code": code, + "me": me, + "redirect_uri": redirect_uri, + "client_id": client_id, + } + ) if not auth: abort(403) - scope = ' '.join(auth['scope']) - payload = dict(me=me, client_id=client_id, scope=scope, ts=datetime.now().timestamp()) - token = JWT.dumps(payload).decode('utf-8') + scope = " ".join(auth["scope"]) + payload = dict( + me=me, client_id=client_id, scope=scope, ts=datetime.now().timestamp() + ) + token = JWT.dumps(payload).decode("utf-8") - return build_auth_resp({'me': me, 'scope': scope, 'access_token': token}) + return build_auth_resp({"me": me, "scope": scope, "access_token": token}) # Token verification - token = request.headers.get('Authorization').replace('Bearer ', '') + token = request.headers.get("Authorization").replace("Bearer ", "") try: payload = JWT.loads(token) except BadSignature: @@ -1431,8 +1536,10 @@ def token_endpoint(): # TODO(tsileo): handle expiration - return build_auth_resp({ - 'me': payload['me'], - 'scope': payload['scope'], - 'client_id': payload['client_id'], - }) + return build_auth_resp( + { + "me": payload["me"], + "scope": payload["scope"], + "client_id": payload["client_id"], + } + ) diff --git a/config.py b/config.py index 9adbff9..c0e1858 100644 --- a/config.py +++ b/config.py @@ -1,13 +1,16 @@ -import subprocess import os -import yaml -from pymongo import MongoClient -import requests -from itsdangerous import JSONWebSignatureSerializer +import subprocess from datetime import datetime -from little_boxes.utils import strtobool -from utils.key import KEY_DIR, get_key, get_secret_key +import requests +import yaml +from itsdangerous import JSONWebSignatureSerializer +from pymongo import MongoClient + +from little_boxes import strtobool +from utils.key import KEY_DIR +from utils.key import get_key +from utils.key import get_secret_key def noop(): diff --git a/dev-requirements.txt b/dev-requirements.txt index 62e71f2..a4ab4e5 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,3 +4,4 @@ html2text pyyaml flake8 mypy +black diff --git a/tasks.py b/tasks.py index a30854a..37b0257 100644 --- a/tasks.py +++ b/tasks.py @@ -1,47 +1,53 @@ -import os import json import logging +import os import random import requests from celery import Celery from requests.exceptions import HTTPError +from config import DB from config import HEADERS from config import ID -from config import DB from config import KEY from config import USER_AGENT from utils.httpsig import HTTPSigAuth -from utils.opengraph import fetch_og_metadata from utils.linked_data_sig import generate_signature - +from utils.opengraph import fetch_og_metadata log = logging.getLogger(__name__) -app = Celery('tasks', broker=os.getenv('MICROBLOGPUB_AMQP_BROKER', 'pyamqp://guest@localhost//')) -SigAuth = HTTPSigAuth(ID+'#main-key', KEY.privkey) +app = Celery( + "tasks", broker=os.getenv("MICROBLOGPUB_AMQP_BROKER", "pyamqp://guest@localhost//") +) +SigAuth = HTTPSigAuth(ID + "#main-key", KEY.privkey) @app.task(bind=True, max_retries=12) def post_to_inbox(self, payload: str, to: str) -> None: try: - log.info('payload=%s', payload) - log.info('generating sig') + log.info("payload=%s", payload) + log.info("generating sig") signed_payload = json.loads(payload) generate_signature(signed_payload, KEY.privkey) - log.info('to=%s', to) - resp = requests.post(to, data=json.dumps(signed_payload), auth=SigAuth, headers={ - 'Content-Type': HEADERS[1], - 'Accept': HEADERS[1], - 'User-Agent': USER_AGENT, - }) - log.info('resp=%s', resp) - log.info('resp_body=%s', resp.text) + log.info("to=%s", to) + resp = requests.post( + to, + data=json.dumps(signed_payload), + auth=SigAuth, + headers={ + "Content-Type": HEADERS[1], + "Accept": HEADERS[1], + "User-Agent": USER_AGENT, + }, + ) + log.info("resp=%s", resp) + log.info("resp_body=%s", resp.text) resp.raise_for_status() except HTTPError as err: - log.exception('request failed') + log.exception("request failed") if 400 >= err.response.status_code >= 499: - log.info('client error, no retry') + log.info("client error, no retry") return self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) @@ -49,11 +55,15 @@ def post_to_inbox(self, payload: str, to: str) -> None: @app.task(bind=True, max_retries=12) def fetch_og(self, col, remote_id): try: - log.info('fetch_og_meta remote_id=%s col=%s', remote_id, col) - if col == 'INBOX': - log.info('%d links saved', fetch_og_metadata(USER_AGENT, DB.inbox, remote_id)) - elif col == 'OUTBOX': - log.info('%d links saved', fetch_og_metadata(USER_AGENT, DB.outbox, remote_id)) + log.info("fetch_og_meta remote_id=%s col=%s", remote_id, col) + if col == "INBOX": + log.info( + "%d links saved", fetch_og_metadata(USER_AGENT, DB.inbox, remote_id) + ) + elif col == "OUTBOX": + log.info( + "%d links saved", fetch_og_metadata(USER_AGENT, DB.outbox, remote_id) + ) except Exception as err: - self.log.exception('failed') + self.log.exception("failed") self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) diff --git a/tests/federation_test.py b/tests/federation_test.py index e050afc..a2dba10 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -1,12 +1,12 @@ -import time import os +import time +from typing import List +from typing import Tuple import requests from html2text import html2text -from utils import activitypub_utils -from typing import Tuple -from typing import List +from utils import activitypub_utils def resp2plaintext(resp): diff --git a/utils/activitypub_utils.py b/utils/activitypub_utils.py index 0275f54..3204237 100644 --- a/utils/activitypub_utils.py +++ b/utils/activitypub_utils.py @@ -1,4 +1,7 @@ -from typing import Optional, Dict, List, Any +from typing import Any +from typing import Dict +from typing import List +from typing import Optional import requests diff --git a/utils/actor_service.py b/utils/actor_service.py index 9982235..f13a6cf 100644 --- a/utils/actor_service.py +++ b/utils/actor_service.py @@ -1,11 +1,11 @@ import logging +from urllib.parse import urlparse import requests -from urllib.parse import urlparse from Crypto.PublicKey import RSA -from .urlutils import check_url from .errors import ActivityNotFoundError +from .urlutils import check_url logger = logging.getLogger(__name__) diff --git a/utils/content_helper.py b/utils/content_helper.py index b254e2b..8ea8cf2 100644 --- a/utils/content_helper.py +++ b/utils/content_helper.py @@ -1,14 +1,21 @@ -import typing -import re +import re +import typing +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Tuple +from typing import Type +from typing import Union -from bleach.linkifier import Linker -from markdown import markdown +from bleach.linkifier import Linker +from markdown import markdown -from utils.webfinger import get_actor_url -from config import USERNAME, BASE_URL, ID -from config import ACTOR_SERVICE - -from typing import List, Optional, Tuple, Dict, Any, Union, Type +from config import ACTOR_SERVICE +from config import BASE_URL +from config import ID +from config import USERNAME +from utils.webfinger import get_actor_url def set_attrs(attrs, new=False): diff --git a/utils/httpsig.py b/utils/httpsig.py index 8437784..609ec3d 100644 --- a/utils/httpsig.py +++ b/utils/httpsig.py @@ -3,19 +3,20 @@ Mastodon instances won't accept requests that are not signed using this scheme. """ -from datetime import datetime -from urllib.parse import urlparse -from typing import Any, Dict, Optional import base64 import hashlib import logging +from datetime import datetime +from typing import Any +from typing import Dict +from typing import Optional +from urllib.parse import urlparse +from Crypto.Hash import SHA256 +from Crypto.Signature import PKCS1_v1_5 from flask import request from requests.auth import AuthBase -from Crypto.Signature import PKCS1_v1_5 -from Crypto.Hash import SHA256 - logger = logging.getLogger(__name__) diff --git a/utils/key.py b/utils/key.py index 18162a5..e7012ae 100644 --- a/utils/key.py +++ b/utils/key.py @@ -1,6 +1,5 @@ -import os import binascii - +import os from typing import Callable from little_boxes.key import Key diff --git a/utils/linked_data_sig.py b/utils/linked_data_sig.py index 834c9bd..b75166e 100644 --- a/utils/linked_data_sig.py +++ b/utils/linked_data_sig.py @@ -1,13 +1,12 @@ -from pyld import jsonld +import base64 import hashlib from datetime import datetime +from typing import Any +from typing import Dict -from Crypto.Signature import PKCS1_v1_5 from Crypto.Hash import SHA256 -import base64 - -from typing import Any, Dict - +from Crypto.Signature import PKCS1_v1_5 +from pyld import jsonld # cache the downloaded "schemas", otherwise the library is super slow # (https://github.com/digitalbazaar/pyld/issues/70) diff --git a/utils/object_service.py b/utils/object_service.py index 1ebc0ce..594fa10 100644 --- a/utils/object_service.py +++ b/utils/object_service.py @@ -1,8 +1,9 @@ -import requests from urllib.parse import urlparse -from .urlutils import check_url +import requests + from .errors import ActivityNotFoundError +from .urlutils import check_url class ObjectService(object): diff --git a/utils/opengraph.py b/utils/opengraph.py index a53c07b..8bafece 100644 --- a/utils/opengraph.py +++ b/utils/opengraph.py @@ -1,11 +1,12 @@ +import ipaddress from urllib.parse import urlparse -import ipaddress import opengraph import requests from bs4 import BeautifulSoup -from .urlutils import is_url_valid, check_url +from .urlutils import check_url +from .urlutils import is_url_valid def links_from_note(note): diff --git a/utils/urlutils.py b/utils/urlutils.py index 99f900d..360d209 100644 --- a/utils/urlutils.py +++ b/utils/urlutils.py @@ -1,7 +1,7 @@ +import ipaddress import logging import os import socket -import ipaddress from urllib.parse import urlparse from . import strtobool diff --git a/utils/webfinger.py b/utils/webfinger.py index 8e6fdc7..344dc01 100644 --- a/utils/webfinger.py +++ b/utils/webfinger.py @@ -1,13 +1,13 @@ -from urllib.parse import urlparse -from typing import Dict, Any -from typing import Optional import logging +from typing import Any +from typing import Dict +from typing import Optional +from urllib.parse import urlparse import requests from .urlutils import check_url - logger = logging.getLogger(__name__) From 4e669620bc78ce99e329b6e5f68c064a5650b46e Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 16 Jun 2018 22:33:51 +0200 Subject: [PATCH 0113/1425] [WIP] Continue migration --- activitypub.py | 35 ++++++++++++++++++++++++++++++++++- app.py | 10 ++++++++-- tasks.py | 9 ++++----- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/activitypub.py b/activitypub.py index 9097d05..4820eaf 100644 --- a/activitypub.py +++ b/activitypub.py @@ -20,6 +20,8 @@ from config import USERNAME from little_boxes import activitypub as ap from little_boxes.backend import Backend from little_boxes.collection import parse_collection as ap_parse_collection +from little_boxes.errors import Error + logger = logging.getLogger(__name__) @@ -41,6 +43,15 @@ def _to_list(data: Union[List[Any], Any]) -> List[Any]: return [data] +def ensure_it_is_me(f): + """Method decorator used to track the events fired during tests.""" + def wrapper(*args, **kwargs): + if args[1].id != MY_PERSON.id: + raise Error('unexpected actor') + return f(*args, **kwargs) + return wrapper + + class MicroblogPubBackend(Backend): def user_agent(self) -> str: return USER_AGENT @@ -51,6 +62,7 @@ class MicroblogPubBackend(Backend): def activity_url(self, obj_id): return f"{BASE_URL}/outbox/{obj_id}" + @ensure_it_is_me def outbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: DB.outbox.insert_one( { @@ -61,6 +73,7 @@ class MicroblogPubBackend(Backend): } ) + @ensure_it_is_me def outbox_is_blocked(self, as_actor: ap.Person, actor_id: str) -> bool: return bool( DB.outbox.find_one( @@ -73,11 +86,14 @@ class MicroblogPubBackend(Backend): ) def fetch_iri(self, iri: str) -> ap.ObjectType: - pass + # FIXME(tsileo): implements caching + return super().fetch_iri(iri) + @ensure_it_is_me def inbox_check_duplicate(self, as_actor: ap.Person, iri: str) -> bool: return bool(DB.inbox.find_one({"remote_id": iri})) + @ensure_it_is_me def inbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: DB.inbox.insert_one( { @@ -88,28 +104,34 @@ class MicroblogPubBackend(Backend): } ) + @ensure_it_is_me def post_to_remote_inbox(self, as_actor: ap.Person, payload: str, to: str) -> None: tasks.post_to_inbox.delay(payload, to) + @ensure_it_is_me def new_follower(self, as_actor: ap.Person, follow: ap.Follow) -> None: remote_actor = follow.get_actor().id if DB.followers.find({"remote_actor": remote_actor}).count() == 0: DB.followers.insert_one({"remote_actor": remote_actor}) + @ensure_it_is_me def undo_new_follower(self, as_actor: ap.Person, follow: ap.Follow) -> None: # TODO(tsileo): update the follow to set undo DB.followers.delete_one({"remote_actor": follow.get_actor().id}) + @ensure_it_is_me def undo_new_following(self, as_actor: ap.Person, follow: ap.Follow) -> None: # TODO(tsileo): update the follow to set undo DB.following.delete_one({"remote_actor": follow.get_object().id}) + @ensure_it_is_me def new_following(self, as_actor: ap.Person, follow: ap.Follow) -> None: remote_actor = follow.get_actor().id if DB.following.find({"remote_actor": remote_actor}).count() == 0: DB.following.insert_one({"remote_actor": remote_actor}) + @ensure_it_is_me def inbox_like(self, as_actor: ap.Person, like: ap.Like) -> None: obj = like.get_object() # Update the meta counter if the object is published by the server @@ -117,6 +139,7 @@ class MicroblogPubBackend(Backend): {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": 1}} ) + @ensure_it_is_me def inbox_undo_like(self, as_actor: ap.Person, like: ap.Like) -> None: obj = like.get_object() # Update the meta counter if the object is published by the server @@ -124,6 +147,7 @@ class MicroblogPubBackend(Backend): {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": -1}} ) + @ensure_it_is_me def outobx_like(self, as_actor: ap.Person, like: ap.Like) -> None: obj = like.get_object() # Unlikely, but an actor can like it's own post @@ -136,6 +160,7 @@ class MicroblogPubBackend(Backend): {"activity.object.id": obj.id}, {"$set": {"meta.liked": like.id}} ) + @ensure_it_is_me def outbox_undo_like(self, as_actor: ap.Person, like: ap.Like) -> None: obj = like.get_object() # Unlikely, but an actor can like it's own post @@ -147,6 +172,7 @@ class MicroblogPubBackend(Backend): {"activity.object.id": obj.id}, {"$set": {"meta.liked": False}} ) + @ensure_it_is_me def inbox_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: if isinstance(announce._data["object"], str) and not announce._data[ "object" @@ -166,6 +192,7 @@ class MicroblogPubBackend(Backend): {"activity.object.id": obj_iri}, {"$inc": {"meta.count_boost": 1}} ) + @ensure_it_is_me def inbox_undo_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: obj = announce.get_object() # Update the meta counter if the object is published by the server @@ -173,18 +200,21 @@ class MicroblogPubBackend(Backend): {"activity.object.id": obj.id}, {"$inc": {"meta.count_boost": -1}} ) + @ensure_it_is_me def outbox_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: obj = announce.get_object() DB.inbox.update_one( {"activity.object.id": obj.id}, {"$set": {"meta.boosted": announce.id}} ) + @ensure_it_is_me def outbox_undo_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: obj = announce.get_object() DB.inbox.update_one( {"activity.object.id": obj.id}, {"$set": {"meta.boosted": False}} ) + @ensure_it_is_me def inbox_delete(self, as_actor: ap.Person, delete: ap.Delete) -> None: DB.inbox.update_one( {"activity.object.id": delete.get_object().id}, @@ -197,12 +227,14 @@ class MicroblogPubBackend(Backend): # TODO(tsileo): also purge the cache if it's a reply of a published activity + @ensure_it_is_me def outbox_delete(self, as_actor: ap.Person, delete: ap.Delete) -> None: DB.outbox.update_one( {"activity.object.id": delete.get_object().id}, {"$set": {"meta.deleted": True}}, ) + @ensure_it_is_me def inbox_update(self, as_actor: ap.Person, update: ap.Update) -> None: obj = update.get_object() if obj.ACTIVITY_TYPE == ap.ActivityType.NOTE: @@ -214,6 +246,7 @@ class MicroblogPubBackend(Backend): # FIXME(tsileo): handle update actor amd inbox_update_note/inbox_update_actor + @ensure_it_is_me def outbox_update(self, as_actor: ap.Person, update: ap.Update) -> None: obj = update._data["object"] diff --git a/app.py b/app.py index e337204..78efbe7 100644 --- a/app.py +++ b/app.py @@ -38,6 +38,7 @@ from werkzeug.utils import secure_filename import activitypub import config from activitypub import embed_collection +from activitypub import MY_PERSON from config import ACTOR_SERVICE from config import ADMIN_API_KEY from config import BASE_URL @@ -55,6 +56,7 @@ from config import USERNAME from config import VERSION from config import _drop_db from config import custom_cache_purge_hook +from little_boxes import activitypub as ap from little_boxes.activitypub import ActivityType from little_boxes.activitypub import clean_activity from little_boxes.errors import BadActivityError @@ -88,6 +90,8 @@ else: SIG_AUTH = HTTPSigAuth(KEY) +OUTBOX = ap.Outbox(MY_PERSON) + def verify_pass(pwd): return bcrypt.verify(pwd, PASS) @@ -377,8 +381,9 @@ def authorize_follow(): if DB.following.find({"remote_actor": actor}).count() > 0: return redirect("/following") - follow = activitypub.Follow(object=actor) - follow.post_to_outbox() + follow = activitypub.Follow(actor=MY_PERSON, object=actor) + OUTBOX.post(follow) + return redirect("/following") @@ -400,6 +405,7 @@ def u2f_register(): ####### # Activity pub routes +# FIXME(tsileo); continue here @app.route("/") diff --git a/tasks.py b/tasks.py index 37b0257..a5c85db 100644 --- a/tasks.py +++ b/tasks.py @@ -9,18 +9,17 @@ from requests.exceptions import HTTPError from config import DB from config import HEADERS -from config import ID from config import KEY from config import USER_AGENT -from utils.httpsig import HTTPSigAuth -from utils.linked_data_sig import generate_signature +from little_boxes.httpsig import HTTPSigAuth +from little_boxes.linked_data_sig import generate_signature from utils.opengraph import fetch_og_metadata log = logging.getLogger(__name__) app = Celery( "tasks", broker=os.getenv("MICROBLOGPUB_AMQP_BROKER", "pyamqp://guest@localhost//") ) -SigAuth = HTTPSigAuth(ID + "#main-key", KEY.privkey) +SigAuth = HTTPSigAuth(KEY) @app.task(bind=True, max_retries=12) @@ -29,7 +28,7 @@ def post_to_inbox(self, payload: str, to: str) -> None: log.info("payload=%s", payload) log.info("generating sig") signed_payload = json.loads(payload) - generate_signature(signed_payload, KEY.privkey) + generate_signature(signed_payload, KEY) log.info("to=%s", to) resp = requests.post( to, From 622006495170aa22cf620d1133adee66287cf893 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 17 Jun 2018 19:21:59 +0200 Subject: [PATCH 0114/1425] More cleanup --- activitypub.py | 9 +- app.py | 87 ++++----- config.py | 76 ++++---- tests/federation_test.py | 389 ++++++++++++++++++++----------------- tests/integration_test.py | 9 +- utils/__init__.py | 6 +- utils/activitypub_utils.py | 68 ------- utils/actor_service.py | 53 +++-- utils/content_helper.py | 65 ------- utils/errors.py | 37 ---- utils/httpsig.py | 95 --------- utils/linked_data_sig.py | 69 ------- utils/object_service.py | 93 ++++++--- utils/opengraph.py | 31 ++- utils/urlutils.py | 47 ----- utils/webfinger.py | 75 ------- 16 files changed, 420 insertions(+), 789 deletions(-) delete mode 100644 utils/activitypub_utils.py delete mode 100644 utils/content_helper.py delete mode 100644 utils/errors.py delete mode 100644 utils/httpsig.py delete mode 100644 utils/linked_data_sig.py delete mode 100644 utils/urlutils.py delete mode 100644 utils/webfinger.py diff --git a/activitypub.py b/activitypub.py index 4820eaf..0ff1e7a 100644 --- a/activitypub.py +++ b/activitypub.py @@ -22,7 +22,6 @@ from little_boxes.backend import Backend from little_boxes.collection import parse_collection as ap_parse_collection from little_boxes.errors import Error - logger = logging.getLogger(__name__) MY_PERSON = ap.Person(**ME) @@ -45,10 +44,12 @@ def _to_list(data: Union[List[Any], Any]) -> List[Any]: def ensure_it_is_me(f): """Method decorator used to track the events fired during tests.""" + def wrapper(*args, **kwargs): if args[1].id != MY_PERSON.id: - raise Error('unexpected actor') + raise Error("unexpected actor") return f(*args, **kwargs) + return wrapper @@ -247,8 +248,8 @@ class MicroblogPubBackend(Backend): # FIXME(tsileo): handle update actor amd inbox_update_note/inbox_update_actor @ensure_it_is_me - def outbox_update(self, as_actor: ap.Person, update: ap.Update) -> None: - obj = update._data["object"] + def outbox_update(self, as_actor: ap.Person, _update: ap.Update) -> None: + obj = _update._data["object"] update_prefix = "activity.object." update: Dict[str, Any] = {"$set": dict(), "$unset": dict()} diff --git a/app.py b/app.py index 78efbe7..33374a2 100644 --- a/app.py +++ b/app.py @@ -30,15 +30,14 @@ from flask import url_for from flask_wtf.csrf import CSRFProtect from html2text import html2text from itsdangerous import BadSignature -from itsdangerous import JSONWebSignatureSerializer from passlib.hash import bcrypt from u2flib_server import u2f from werkzeug.utils import secure_filename import activitypub import config -from activitypub import embed_collection from activitypub import MY_PERSON +from activitypub import embed_collection from config import ACTOR_SERVICE from config import ADMIN_API_KEY from config import BASE_URL @@ -59,16 +58,14 @@ from config import custom_cache_purge_hook from little_boxes import activitypub as ap from little_boxes.activitypub import ActivityType from little_boxes.activitypub import clean_activity -from little_boxes.errors import BadActivityError +from little_boxes.content_helper import parse_markdown +from little_boxes.errors import ActivityNotFoundError from little_boxes.errors import Error -from little_boxes.errors import UnexpectedActivityTypeError +from little_boxes.errors import NotFromOutboxError from little_boxes.httpsig import HTTPSigAuth from little_boxes.httpsig import verify_request from little_boxes.webfinger import get_actor_url from little_boxes.webfinger import get_remote_follow_template -from utils.content_helper import parse_markdown -from utils.errors import ActivityNotFoundError -from utils.errors import NotFromOutboxError from utils.key import get_secret_key app = Flask(__name__) @@ -91,6 +88,7 @@ else: SIG_AUTH = HTTPSigAuth(KEY) OUTBOX = ap.Outbox(MY_PERSON) +INBOX = ap.Inbox(MY_PERSON) def verify_pass(pwd): @@ -405,7 +403,6 @@ def u2f_register(): ####### # Activity pub routes -# FIXME(tsileo); continue here @app.route("/") @@ -726,12 +723,8 @@ def outbox(): data = request.get_json(force=True) print(data) - activity = activitypub.parse_activity(data) - - if activity.type_enum == ActivityType.NOTE: - activity = activity.build_create() - - activity.post_to_outbox() + activity = ap.parse_activity(data) + OUTBOX.post(activity) # Purge the cache if a custom hook is set, as new content was published custom_cache_purge_hook() @@ -743,7 +736,7 @@ def outbox(): def outbox_detail(item_id): doc = DB.outbox.find_one({"id": item_id}) if doc["meta"].get("deleted", False): - obj = activitypub.parse_activity(doc["activity"]) + obj = ap.parse_activity(doc["activity"]) resp = jsonify(**obj.get_object().get_tombstone()) resp.status_code = 410 return resp @@ -770,8 +763,8 @@ def outbox_activity_replies(item_id): data = DB.outbox.find_one({"id": item_id, "meta.deleted": False}) if not data: abort(404) - obj = activitypub.parse_activity(data["activity"]) - if obj.type_enum != ActivityType.CREATE: + obj = ap.parse_activity(data["activity"]) + if obj.ACTIVITY_TYPE != ActivityType.CREATE: abort(404) q = { @@ -800,8 +793,8 @@ def outbox_activity_likes(item_id): data = DB.outbox.find_one({"id": item_id, "meta.deleted": False}) if not data: abort(404) - obj = activitypub.parse_activity(data["activity"]) - if obj.type_enum != ActivityType.CREATE: + obj = ap.parse_activity(data["activity"]) + if obj.ACTIVITY_TYPE != ActivityType.CREATE: abort(404) q = { @@ -833,8 +826,8 @@ def outbox_activity_shares(item_id): data = DB.outbox.find_one({"id": item_id, "meta.deleted": False}) if not data: abort(404) - obj = activitypub.parse_activity(data["activity"]) - if obj.type_enum != ActivityType.CREATE: + obj = ap.parse_activity(data["activity"]) + if obj.ACTIVITY_TYPE != ActivityType.CREATE: abort(404) q = { @@ -890,9 +883,9 @@ def new(): if not data: abort(400) - reply = activitypub.parse_activity(data["activity"]) + reply = ap.parse_activity(data["activity"]) reply_id = reply.id - if reply.type_enum == ActivityType.CREATE: + if reply.ACTIVITY_TYPE == ActivityType.CREATE: reply_id = reply.get_object().id actor = reply.get_actor() domain = urlparse(actor.id).netloc @@ -972,12 +965,10 @@ def _user_api_arg(key: str, **kwargs): def _user_api_get_note(from_outbox: bool = False): oid = _user_api_arg("id") - note = activitypub.parse_activity( - OBJECT_SERVICE.get(oid), expected=ActivityType.NOTE - ) + note = ap.parse_activity(OBJECT_SERVICE.get(oid), expected=ActivityType.NOTE) if from_outbox and not note.id.startswith(ID): raise NotFromOutboxError( - f"cannot delete {note.id}, id must be owned by the server" + f"cannot load {note.id}, id must be owned by the server" ) return note @@ -1000,7 +991,7 @@ def api_delete(): note = _user_api_get_note(from_outbox=True) delete = note.build_delete() - delete.post_to_outbox() + OUTBOX.post(delete) return _user_api_response(activity=delete.id) @@ -1011,7 +1002,7 @@ def api_boost(): note = _user_api_get_note() announce = note.build_announce() - announce.post_to_outbox() + OUTBOX.post(announce) return _user_api_response(activity=announce.id) @@ -1022,7 +1013,7 @@ def api_like(): note = _user_api_get_note() like = note.build_like() - like.post_to_outbox() + OUTBOX.post(like) return _user_api_response(activity=like.id) @@ -1035,10 +1026,10 @@ def api_undo(): if not doc: raise ActivityNotFoundError(f"cannot found {oid}") - obj = activitypub.parse_activity(doc.get("activity")) + obj = ap.parse_activity(doc.get("activity")) # FIXME(tsileo): detect already undo-ed and make this API call idempotent undo = obj.build_undo() - undo.post_to_outbox() + OUTBOX.post(undo) return _user_api_response(activity=undo.id) @@ -1116,7 +1107,7 @@ def inbox(): data = request.get_json(force=True) logger.debug(f"req_headers={request.headers}") logger.debug(f"raw_data={data}") - try: + """try: if not verify_request(ACTOR_SERVICE): raise Exception("failed to verify request") except Exception: @@ -1136,10 +1127,10 @@ def inbox(): } ), ) - - activity = activitypub.parse_activity(data) + """ + activity = ap.parse_activity(data) logger.debug(f"inbox activity={activity}/{data}") - activity.process_from_inbox() + INBOX.post(activity) return Response(status=201) @@ -1185,9 +1176,10 @@ def api_upload(): print(attachment) content = request.args.get("content") to = request.args.get("to") - note = activitypub.Note( + note = ap.Note( + actor=MY_PERSON, cc=[ID + "/followers"], - to=[to if to else config.AS_PUBLIC], + to=[to if to else ap.AS_PUBLIC], content=content, # TODO(tsileo): handle markdown attachment=attachment, ) @@ -1196,7 +1188,7 @@ def api_upload(): create = note.build_create() print(create) print(create.to_dict()) - create.post_to_outbox() + OUTBOX.post(create) print("posted") return Response(status=201, response="OK") @@ -1220,23 +1212,24 @@ def api_new_note(): cc = [ID + "/followers"] if _reply: - reply = activitypub.parse_activity(OBJECT_SERVICE.get(_reply)) + reply = ap.parse_activity(OBJECT_SERVICE.get(_reply)) cc.append(reply.attributedTo) for tag in tags: if tag["type"] == "Mention": cc.append(tag["href"]) - note = activitypub.Note( + note = ap.Note( + actor=MY_PERSON, cc=list(set(cc)), - to=[to if to else config.AS_PUBLIC], + to=[to if to else ap.AS_PUBLIC], content=content, tag=tags, source={"mediaType": "text/markdown", "content": source}, inReplyTo=reply.id if reply else None, ) create = note.build_create() - create.post_to_outbox() + OUTBOX.post(create) return _user_api_response(activity=create.id) @@ -1263,8 +1256,8 @@ def api_block(): if existing: return _user_api_response(activity=existing["activity"]["id"]) - block = activitypub.Block(object=actor) - block.post_to_outbox() + block = ap.Block(actor=MY_PERSON, object=actor) + OUTBOX.post(block) return _user_api_response(activity=block.id) @@ -1278,8 +1271,8 @@ def api_follow(): if existing: return _user_api_response(activity=existing["activity"]["id"]) - follow = activitypub.Follow(object=actor) - follow.post_to_outbox() + follow = ap.Follow(actor=MY_PERSON, object=actor) + OUTBOX.post(follow) return _user_api_response(activity=follow.id) diff --git a/config.py b/config.py index c0e1858..8478186 100644 --- a/config.py +++ b/config.py @@ -23,48 +23,49 @@ try: except ModuleNotFoundError: custom_cache_purge_hook = noop -VERSION = subprocess.check_output(['git', 'describe', '--always']).split()[0].decode('utf-8') +VERSION = ( + subprocess.check_output(["git", "describe", "--always"]).split()[0].decode("utf-8") +) -DEBUG_MODE = strtobool(os.getenv('MICROBLOGPUB_DEBUG', 'false')) +DEBUG_MODE = strtobool(os.getenv("MICROBLOGPUB_DEBUG", "false")) -CTX_AS = 'https://www.w3.org/ns/activitystreams' -CTX_SECURITY = 'https://w3id.org/security/v1' -AS_PUBLIC = 'https://www.w3.org/ns/activitystreams#Public' +CTX_AS = "https://www.w3.org/ns/activitystreams" +CTX_SECURITY = "https://w3id.org/security/v1" +AS_PUBLIC = "https://www.w3.org/ns/activitystreams#Public" HEADERS = [ - 'application/activity+json', - 'application/ld+json;profile=https://www.w3.org/ns/activitystreams', + "application/activity+json", + "application/ld+json;profile=https://www.w3.org/ns/activitystreams", 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', - 'application/ld+json', + "application/ld+json", ] -with open(os.path.join(KEY_DIR, 'me.yml')) as f: +with open(os.path.join(KEY_DIR, "me.yml")) as f: conf = yaml.load(f) - USERNAME = conf['username'] - NAME = conf['name'] - DOMAIN = conf['domain'] - SCHEME = 'https' if conf.get('https', True) else 'http' - BASE_URL = SCHEME + '://' + DOMAIN + USERNAME = conf["username"] + NAME = conf["name"] + DOMAIN = conf["domain"] + SCHEME = "https" if conf.get("https", True) else "http" + BASE_URL = SCHEME + "://" + DOMAIN ID = BASE_URL - SUMMARY = conf['summary'] - ICON_URL = conf['icon_url'] - PASS = conf['pass'] - PUBLIC_INSTANCES = conf.get('public_instances', []) + SUMMARY = conf["summary"] + ICON_URL = conf["icon_url"] + PASS = conf["pass"] + PUBLIC_INSTANCES = conf.get("public_instances", []) # TODO(tsileo): choose dark/light style - THEME_COLOR = conf.get('theme_color') + THEME_COLOR = conf.get("theme_color") USER_AGENT = ( - f'{requests.utils.default_user_agent()} ' - f'(microblog.pub/{VERSION}; +{BASE_URL})' + f"{requests.utils.default_user_agent()} " f"(microblog.pub/{VERSION}; +{BASE_URL})" ) mongo_client = MongoClient( - host=[os.getenv('MICROBLOGPUB_MONGODB_HOST', 'localhost:27017')], + host=[os.getenv("MICROBLOGPUB_MONGODB_HOST", "localhost:27017")] ) -DB_NAME = '{}_{}'.format(USERNAME, DOMAIN.replace('.', '_')) +DB_NAME = "{}_{}".format(USERNAME, DOMAIN.replace(".", "_")) DB = mongo_client[DB_NAME] @@ -78,37 +79,32 @@ def _drop_db(): KEY = get_key(ID, USERNAME, DOMAIN) -JWT_SECRET = get_secret_key('jwt') +JWT_SECRET = get_secret_key("jwt") JWT = JSONWebSignatureSerializer(JWT_SECRET) def _admin_jwt_token() -> str: - return JWT.dumps({'me': 'ADMIN', 'ts': datetime.now().timestamp()}).decode('utf-8') # type: ignore + return JWT.dumps({"me": "ADMIN", "ts": datetime.now().timestamp()}).decode( + "utf-8" + ) # type: ignore -ADMIN_API_KEY = get_secret_key('admin_api_key', _admin_jwt_token) +ADMIN_API_KEY = get_secret_key("admin_api_key", _admin_jwt_token) ME = { - "@context": [ - CTX_AS, - CTX_SECURITY, - ], + "@context": [CTX_AS, CTX_SECURITY], "type": "Person", "id": ID, - "following": ID+"/following", - "followers": ID+"/followers", - "liked": ID+"/liked", - "inbox": ID+"/inbox", - "outbox": ID+"/outbox", + "following": ID + "/following", + "followers": ID + "/followers", + "liked": ID + "/liked", + "inbox": ID + "/inbox", + "outbox": ID + "/outbox", "preferredUsername": USERNAME, "name": NAME, "summary": SUMMARY, "endpoints": {}, "url": ID, - "icon": { - "mediaType": "image/png", - "type": "Image", - "url": ICON_URL, - }, + "icon": {"mediaType": "image/png", "type": "Image", "url": ICON_URL}, "publicKey": KEY.to_dict(), } diff --git a/tests/federation_test.py b/tests/federation_test.py index a2dba10..1bdda92 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -22,10 +22,13 @@ class Instance(object): self.docker_url = docker_url or host_url self._create_delay = 10 with open( - os.path.join(os.path.dirname(os.path.abspath(__file__)), f'fixtures/{name}/config/admin_api_key.key') + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + f"fixtures/{name}/config/admin_api_key.key", + ) ) as f: api_key = f.read() - self._auth_headers = {'Authorization': f'Bearer {api_key}'} + self._auth_headers = {"Authorization": f"Bearer {api_key}"} def _do_req(self, url, headers): """Used to parse collection.""" @@ -36,19 +39,21 @@ class Instance(object): def _parse_collection(self, payload=None, url=None): """Parses a collection (go through all the pages).""" - return activitypub_utils.parse_collection(url=url, payload=payload, do_req=self._do_req) + return activitypub_utils.parse_collection( + url=url, payload=payload, do_req=self._do_req + ) def ping(self): """Ensures the homepage is reachable.""" - resp = requests.get(f'{self.host_url}/') + resp = requests.get(f"{self.host_url}/") resp.raise_for_status() assert resp.status_code == 200 def debug(self): """Returns the debug infos (number of items in the inbox/outbox.""" resp = requests.get( - f'{self.host_url}/api/debug', - headers={**self._auth_headers, 'Accept': 'application/json'}, + f"{self.host_url}/api/debug", + headers={**self._auth_headers, "Accept": "application/json"}, ) resp.raise_for_status() @@ -57,8 +62,8 @@ class Instance(object): def drop_db(self): """Drops the MongoDB DB.""" resp = requests.delete( - f'{self.host_url}/api/debug', - headers={**self._auth_headers, 'Accept': 'application/json'}, + f"{self.host_url}/api/debug", + headers={**self._auth_headers, "Accept": "application/json"}, ) resp.raise_for_status() @@ -68,100 +73,92 @@ class Instance(object): """Blocks an actor.""" # Instance1 follows instance2 resp = requests.post( - f'{self.host_url}/api/block', - params={'actor': actor_url}, + f"{self.host_url}/api/block", + params={"actor": actor_url}, headers=self._auth_headers, ) assert resp.status_code == 201 # We need to wait for the Follow/Accept dance - time.sleep(self._create_delay/2) - return resp.json().get('activity') + time.sleep(self._create_delay / 2) + return resp.json().get("activity") - def follow(self, instance: 'Instance') -> str: + def follow(self, instance: "Instance") -> str: """Follows another instance.""" # Instance1 follows instance2 resp = requests.post( - f'{self.host_url}/api/follow', - json={'actor': instance.docker_url}, + f"{self.host_url}/api/follow", + json={"actor": instance.docker_url}, headers=self._auth_headers, ) assert resp.status_code == 201 # We need to wait for the Follow/Accept dance time.sleep(self._create_delay) - return resp.json().get('activity') + return resp.json().get("activity") def new_note(self, content, reply=None) -> str: """Creates a new note.""" - params = {'content': content} + params = {"content": content} if reply: - params['reply'] = reply + params["reply"] = reply resp = requests.post( - f'{self.host_url}/api/new_note', - json=params, - headers=self._auth_headers, + f"{self.host_url}/api/new_note", json=params, headers=self._auth_headers ) assert resp.status_code == 201 time.sleep(self._create_delay) - return resp.json().get('activity') + return resp.json().get("activity") def boost(self, oid: str) -> str: """Creates an Announce activity.""" resp = requests.post( - f'{self.host_url}/api/boost', - json={'id': oid}, - headers=self._auth_headers, + f"{self.host_url}/api/boost", json={"id": oid}, headers=self._auth_headers ) assert resp.status_code == 201 time.sleep(self._create_delay) - return resp.json().get('activity') + return resp.json().get("activity") def like(self, oid: str) -> str: """Creates a Like activity.""" resp = requests.post( - f'{self.host_url}/api/like', - json={'id': oid}, - headers=self._auth_headers, + f"{self.host_url}/api/like", json={"id": oid}, headers=self._auth_headers ) assert resp.status_code == 201 time.sleep(self._create_delay) - return resp.json().get('activity') + return resp.json().get("activity") def delete(self, oid: str) -> str: """Creates a Delete activity.""" resp = requests.post( - f'{self.host_url}/api/note/delete', - json={'id': oid}, + f"{self.host_url}/api/note/delete", + json={"id": oid}, headers=self._auth_headers, ) assert resp.status_code == 201 time.sleep(self._create_delay) - return resp.json().get('activity') + return resp.json().get("activity") def undo(self, oid: str) -> str: """Creates a Undo activity.""" resp = requests.post( - f'{self.host_url}/api/undo', - json={'id': oid}, - headers=self._auth_headers, + f"{self.host_url}/api/undo", json={"id": oid}, headers=self._auth_headers ) assert resp.status_code == 201 # We need to wait for the Follow/Accept dance time.sleep(self._create_delay) - return resp.json().get('activity') + return resp.json().get("activity") def followers(self) -> List[str]: """Parses the followers collection.""" resp = requests.get( - f'{self.host_url}/followers', - headers={'Accept': 'application/activity+json'}, + f"{self.host_url}/followers", + headers={"Accept": "application/activity+json"}, ) resp.raise_for_status() @@ -172,8 +169,8 @@ class Instance(object): def following(self): """Parses the following collection.""" resp = requests.get( - f'{self.host_url}/following', - headers={'Accept': 'application/activity+json'}, + f"{self.host_url}/following", + headers={"Accept": "application/activity+json"}, ) resp.raise_for_status() @@ -184,8 +181,8 @@ class Instance(object): def outbox(self): """Returns the instance outbox.""" resp = requests.get( - f'{self.host_url}/following', - headers={'Accept': 'application/activity+json'}, + f"{self.host_url}/following", + headers={"Accept": "application/activity+json"}, ) resp.raise_for_status() return resp.json() @@ -194,7 +191,7 @@ class Instance(object): """Fetches a specific item from the instance outbox.""" resp = requests.get( aid.replace(self.docker_url, self.host_url), - headers={'Accept': 'application/activity+json'}, + headers={"Accept": "application/activity+json"}, ) resp.raise_for_status() return resp.json() @@ -202,8 +199,8 @@ class Instance(object): def stream_jsonfeed(self): """Returns the "stream"'s JSON feed.""" resp = requests.get( - f'{self.host_url}/api/stream', - headers={**self._auth_headers, 'Accept': 'application/json'}, + f"{self.host_url}/api/stream", + headers={**self._auth_headers, "Accept": "application/json"}, ) resp.raise_for_status() return resp.json() @@ -211,10 +208,14 @@ class Instance(object): def _instances() -> Tuple[Instance, Instance]: """Initializes the client for the two test instances.""" - instance1 = Instance('instance1', 'http://localhost:5006', 'http://instance1_web_1:5005') + instance1 = Instance( + "instance1", "http://localhost:5006", "http://instance1_web_1:5005" + ) instance1.ping() - instance2 = Instance('instance2', 'http://localhost:5007', 'http://instance2_web_1:5005') + instance2 = Instance( + "instance2", "http://localhost:5007", "http://instance2_web_1:5005" + ) instance2.ping() # Return the DB @@ -230,12 +231,12 @@ def test_follow() -> None: # Instance1 follows instance2 instance1.follow(instance2) instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 1 # An Accept activity should be there - assert instance1_debug['outbox'] == 1 # We've sent a Follow activity + assert instance1_debug["inbox"] == 1 # An Accept activity should be there + assert instance1_debug["outbox"] == 1 # We've sent a Follow activity instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 1 # An Follow activity should be there - assert instance2_debug['outbox'] == 1 # We've sent a Accept activity + assert instance2_debug["inbox"] == 1 # An Follow activity should be there + assert instance2_debug["outbox"] == 1 # We've sent a Accept activity assert instance2.followers() == [instance1.docker_url] assert instance1.following() == [instance2.docker_url] @@ -247,12 +248,12 @@ def test_follow_unfollow(): # Instance1 follows instance2 follow_id = instance1.follow(instance2) instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 1 # An Accept activity should be there - assert instance1_debug['outbox'] == 1 # We've sent a Follow activity + assert instance1_debug["inbox"] == 1 # An Accept activity should be there + assert instance1_debug["outbox"] == 1 # We've sent a Follow activity instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 1 # An Follow activity should be there - assert instance2_debug['outbox'] == 1 # We've sent a Accept activity + assert instance2_debug["inbox"] == 1 # An Follow activity should be there + assert instance2_debug["outbox"] == 1 # We've sent a Accept activity assert instance2.followers() == [instance1.docker_url] assert instance1.following() == [instance2.docker_url] @@ -263,12 +264,12 @@ def test_follow_unfollow(): assert instance1.following() == [] instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 1 # An Accept activity should be there - assert instance1_debug['outbox'] == 2 # We've sent a Follow and a Undo activity + assert instance1_debug["inbox"] == 1 # An Accept activity should be there + assert instance1_debug["outbox"] == 2 # We've sent a Follow and a Undo activity instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 2 # An Follow and Undo activity should be there - assert instance2_debug['outbox'] == 1 # We've sent a Accept activity + assert instance2_debug["inbox"] == 2 # An Follow and Undo activity should be there + assert instance2_debug["outbox"] == 1 # We've sent a Accept activity def test_post_content(): @@ -279,17 +280,19 @@ def test_post_content(): instance2.follow(instance1) inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 - create_id = instance1.new_note('hello') + create_id = instance1.new_note("hello") instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + assert ( + instance2_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert instance2_debug["outbox"] == 2 # We've sent a Accept and a Follow activity # Ensure the post is visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 1 - assert inbox_stream['items'][0]['id'] == create_id + assert len(inbox_stream["items"]) == 1 + assert inbox_stream["items"][0]["id"] == create_id def test_block_and_post_content(): @@ -300,18 +303,22 @@ def test_block_and_post_content(): instance2.follow(instance1) inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 instance2.block(instance1.docker_url) - instance1.new_note('hello') + instance1.new_note("hello") instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 2 # An Follow, Accept activity should be there, Create should have been dropped - assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow activity + the Block activity + assert ( + instance2_debug["inbox"] == 2 + ) # An Follow, Accept activity should be there, Create should have been dropped + assert ( + instance2_debug["outbox"] == 3 + ) # We've sent a Accept and a Follow activity + the Block activity # Ensure the post is not visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 def test_post_content_and_delete(): @@ -322,26 +329,30 @@ def test_post_content_and_delete(): instance2.follow(instance1) inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 - create_id = instance1.new_note('hello') + create_id = instance1.new_note("hello") instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + assert ( + instance2_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert instance2_debug["outbox"] == 2 # We've sent a Accept and a Follow activity # Ensure the post is visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 1 - assert inbox_stream['items'][0]['id'] == create_id + assert len(inbox_stream["items"]) == 1 + assert inbox_stream["items"][0]["id"] == create_id - instance1.delete(f'{create_id}/activity') + instance1.delete(f"{create_id}/activity") instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 4 # An Follow, Accept and Create and Delete activity should be there - assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + assert ( + instance2_debug["inbox"] == 4 + ) # An Follow, Accept and Create and Delete activity should be there + assert instance2_debug["outbox"] == 2 # We've sent a Accept and a Follow activity # Ensure the post has been delete from instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 def test_post_content_and_like(): @@ -351,26 +362,26 @@ def test_post_content_and_like(): instance1.follow(instance2) instance2.follow(instance1) - create_id = instance1.new_note('hello') + create_id = instance1.new_note("hello") # Ensure the post is visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 1 - assert inbox_stream['items'][0]['id'] == create_id + assert len(inbox_stream["items"]) == 1 + assert inbox_stream["items"][0]["id"] == create_id # Now, instance2 like the note - like_id = instance2.like(f'{create_id}/activity') + like_id = instance2.like(f"{create_id}/activity") instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 3 # Follow, Accept and Like - assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + assert instance1_debug["inbox"] == 3 # Follow, Accept and Like + assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create - note = instance1.outbox_get(f'{create_id}/activity') - assert 'likes' in note - assert note['likes']['totalItems'] == 1 - likes = instance1._parse_collection(url=note['likes']['first']) + note = instance1.outbox_get(f"{create_id}/activity") + assert "likes" in note + assert note["likes"]["totalItems"] == 1 + likes = instance1._parse_collection(url=note["likes"]["first"]) assert len(likes) == 1 - assert likes[0]['id'] == like_id + assert likes[0]["id"] == like_id def test_post_content_and_like_unlike() -> None: @@ -380,36 +391,36 @@ def test_post_content_and_like_unlike() -> None: instance1.follow(instance2) instance2.follow(instance1) - create_id = instance1.new_note('hello') + create_id = instance1.new_note("hello") # Ensure the post is visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 1 - assert inbox_stream['items'][0]['id'] == create_id + assert len(inbox_stream["items"]) == 1 + assert inbox_stream["items"][0]["id"] == create_id # Now, instance2 like the note - like_id = instance2.like(f'{create_id}/activity') + like_id = instance2.like(f"{create_id}/activity") instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 3 # Follow, Accept and Like - assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + assert instance1_debug["inbox"] == 3 # Follow, Accept and Like + assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create - note = instance1.outbox_get(f'{create_id}/activity') - assert 'likes' in note - assert note['likes']['totalItems'] == 1 - likes = instance1._parse_collection(url=note['likes']['first']) + note = instance1.outbox_get(f"{create_id}/activity") + assert "likes" in note + assert note["likes"]["totalItems"] == 1 + likes = instance1._parse_collection(url=note["likes"]["first"]) assert len(likes) == 1 - assert likes[0]['id'] == like_id + assert likes[0]["id"] == like_id instance2.undo(like_id) instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 4 # Follow, Accept and Like and Undo - assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + assert instance1_debug["inbox"] == 4 # Follow, Accept and Like and Undo + assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create - note = instance1.outbox_get(f'{create_id}/activity') - assert 'likes' in note - assert note['likes']['totalItems'] == 0 + note = instance1.outbox_get(f"{create_id}/activity") + assert "likes" in note + assert note["likes"]["totalItems"] == 0 def test_post_content_and_boost() -> None: @@ -419,26 +430,26 @@ def test_post_content_and_boost() -> None: instance1.follow(instance2) instance2.follow(instance1) - create_id = instance1.new_note('hello') + create_id = instance1.new_note("hello") # Ensure the post is visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 1 - assert inbox_stream['items'][0]['id'] == create_id + assert len(inbox_stream["items"]) == 1 + assert inbox_stream["items"][0]["id"] == create_id # Now, instance2 like the note - boost_id = instance2.boost(f'{create_id}/activity') + boost_id = instance2.boost(f"{create_id}/activity") instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 3 # Follow, Accept and Announce - assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + assert instance1_debug["inbox"] == 3 # Follow, Accept and Announce + assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create - note = instance1.outbox_get(f'{create_id}/activity') - assert 'shares' in note - assert note['shares']['totalItems'] == 1 - shares = instance1._parse_collection(url=note['shares']['first']) + note = instance1.outbox_get(f"{create_id}/activity") + assert "shares" in note + assert note["shares"]["totalItems"] == 1 + shares = instance1._parse_collection(url=note["shares"]["first"]) assert len(shares) == 1 - assert shares[0]['id'] == boost_id + assert shares[0]["id"] == boost_id def test_post_content_and_boost_unboost() -> None: @@ -448,36 +459,36 @@ def test_post_content_and_boost_unboost() -> None: instance1.follow(instance2) instance2.follow(instance1) - create_id = instance1.new_note('hello') + create_id = instance1.new_note("hello") # Ensure the post is visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 1 - assert inbox_stream['items'][0]['id'] == create_id + assert len(inbox_stream["items"]) == 1 + assert inbox_stream["items"][0]["id"] == create_id # Now, instance2 like the note - boost_id = instance2.boost(f'{create_id}/activity') + boost_id = instance2.boost(f"{create_id}/activity") instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 3 # Follow, Accept and Announce - assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + assert instance1_debug["inbox"] == 3 # Follow, Accept and Announce + assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create - note = instance1.outbox_get(f'{create_id}/activity') - assert 'shares' in note - assert note['shares']['totalItems'] == 1 - shares = instance1._parse_collection(url=note['shares']['first']) + note = instance1.outbox_get(f"{create_id}/activity") + assert "shares" in note + assert note["shares"]["totalItems"] == 1 + shares = instance1._parse_collection(url=note["shares"]["first"]) assert len(shares) == 1 - assert shares[0]['id'] == boost_id + assert shares[0]["id"] == boost_id instance2.undo(boost_id) instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 4 # Follow, Accept and Announce and Undo - assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + assert instance1_debug["inbox"] == 4 # Follow, Accept and Announce and Undo + assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create - note = instance1.outbox_get(f'{create_id}/activity') - assert 'shares' in note - assert note['shares']['totalItems'] == 0 + note = instance1.outbox_get(f"{create_id}/activity") + assert "shares" in note + assert note["shares"]["totalItems"] == 0 def test_post_content_and_post_reply() -> None: @@ -488,40 +499,50 @@ def test_post_content_and_post_reply() -> None: instance2.follow(instance1) inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 - instance1_create_id = instance1.new_note('hello') + instance1_create_id = instance1.new_note("hello") instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + assert ( + instance2_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert instance2_debug["outbox"] == 2 # We've sent a Accept and a Follow activity # Ensure the post is visible in instance2's stream instance2_inbox_stream = instance2.stream_jsonfeed() - assert len(instance2_inbox_stream['items']) == 1 - assert instance2_inbox_stream['items'][0]['id'] == instance1_create_id + assert len(instance2_inbox_stream["items"]) == 1 + assert instance2_inbox_stream["items"][0]["id"] == instance1_create_id instance2_create_id = instance2.new_note( - f'hey @instance1@{instance1.docker_url}', - reply=f'{instance1_create_id}/activity', + f"hey @instance1@{instance1.docker_url}", + reply=f"{instance1_create_id}/activity", ) instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity + assert ( + instance2_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert ( + instance2_debug["outbox"] == 3 + ) # We've sent a Accept and a Follow and a Create activity instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance1_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity + assert ( + instance1_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert ( + instance1_debug["outbox"] == 3 + ) # We've sent a Accept and a Follow and a Create activity instance1_inbox_stream = instance1.stream_jsonfeed() - assert len(instance1_inbox_stream['items']) == 1 - assert instance1_inbox_stream['items'][0]['id'] == instance2_create_id + assert len(instance1_inbox_stream["items"]) == 1 + assert instance1_inbox_stream["items"][0]["id"] == instance2_create_id - instance1_note = instance1.outbox_get(f'{instance1_create_id}/activity') - assert 'replies' in instance1_note - assert instance1_note['replies']['totalItems'] == 1 - replies = instance1._parse_collection(url=instance1_note['replies']['first']) + instance1_note = instance1.outbox_get(f"{instance1_create_id}/activity") + assert "replies" in instance1_note + assert instance1_note["replies"]["totalItems"] == 1 + replies = instance1._parse_collection(url=instance1_note["replies"]["first"]) assert len(replies) == 1 - assert replies[0]['id'] == f'{instance2_create_id}/activity' + assert replies[0]["id"] == f"{instance2_create_id}/activity" def test_post_content_and_post_reply_and_delete() -> None: @@ -532,44 +553,58 @@ def test_post_content_and_post_reply_and_delete() -> None: instance2.follow(instance1) inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 - instance1_create_id = instance1.new_note('hello') + instance1_create_id = instance1.new_note("hello") instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + assert ( + instance2_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert instance2_debug["outbox"] == 2 # We've sent a Accept and a Follow activity # Ensure the post is visible in instance2's stream instance2_inbox_stream = instance2.stream_jsonfeed() - assert len(instance2_inbox_stream['items']) == 1 - assert instance2_inbox_stream['items'][0]['id'] == instance1_create_id + assert len(instance2_inbox_stream["items"]) == 1 + assert instance2_inbox_stream["items"][0]["id"] == instance1_create_id instance2_create_id = instance2.new_note( - f'hey @instance1@{instance1.docker_url}', - reply=f'{instance1_create_id}/activity', + f"hey @instance1@{instance1.docker_url}", + reply=f"{instance1_create_id}/activity", ) instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity + assert ( + instance2_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert ( + instance2_debug["outbox"] == 3 + ) # We've sent a Accept and a Follow and a Create activity instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance1_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity + assert ( + instance1_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert ( + instance1_debug["outbox"] == 3 + ) # We've sent a Accept and a Follow and a Create activity instance1_inbox_stream = instance1.stream_jsonfeed() - assert len(instance1_inbox_stream['items']) == 1 - assert instance1_inbox_stream['items'][0]['id'] == instance2_create_id + assert len(instance1_inbox_stream["items"]) == 1 + assert instance1_inbox_stream["items"][0]["id"] == instance2_create_id - instance1_note = instance1.outbox_get(f'{instance1_create_id}/activity') - assert 'replies' in instance1_note - assert instance1_note['replies']['totalItems'] == 1 + instance1_note = instance1.outbox_get(f"{instance1_create_id}/activity") + assert "replies" in instance1_note + assert instance1_note["replies"]["totalItems"] == 1 - instance2.delete(f'{instance2_create_id}/activity') + instance2.delete(f"{instance2_create_id}/activity") instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 4 # An Follow, Accept and Create and Delete activity should be there - assert instance1_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity + assert ( + instance1_debug["inbox"] == 4 + ) # An Follow, Accept and Create and Delete activity should be there + assert ( + instance1_debug["outbox"] == 3 + ) # We've sent a Accept and a Follow and a Create activity - instance1_note = instance1.outbox_get(f'{instance1_create_id}/activity') - assert 'replies' in instance1_note - assert instance1_note['replies']['totalItems'] == 0 + instance1_note = instance1.outbox_get(f"{instance1_create_id}/activity") + assert "replies" in instance1_note + assert instance1_note["replies"]["totalItems"] == 0 diff --git a/tests/integration_test.py b/tests/integration_test.py index 4270b4b..dbfe19b 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -9,7 +9,10 @@ from html2text import html2text def config(): """Return the current config as a dict.""" import yaml - with open(os.path.join(os.path.dirname(__file__), '..', 'config/me.yml'), 'rb') as f: + + with open( + os.path.join(os.path.dirname(__file__), "..", "config/me.yml"), "rb" + ) as f: yield yaml.load(f) @@ -20,9 +23,9 @@ def resp2plaintext(resp): def test_ping_homepage(config): """Ensure the homepage is accessible.""" - resp = requests.get('http://localhost:5005') + resp = requests.get("http://localhost:5005") resp.raise_for_status() assert resp.status_code == 200 body = resp2plaintext(resp) - assert config['name'] in body + assert config["name"] in body assert f"@{config['username']}@{config['domain']}" in body diff --git a/utils/__init__.py b/utils/__init__.py index c30c37d..cdf368d 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -4,9 +4,9 @@ logger = logging.getLogger(__name__) def strtobool(s: str) -> bool: - if s in ['y', 'yes', 'true', 'on', '1']: + if s in ["y", "yes", "true", "on", "1"]: return True - if s in ['n', 'no', 'false', 'off', '0']: + if s in ["n", "no", "false", "off", "0"]: return False - raise ValueError(f'cannot convert {s} to bool') + raise ValueError(f"cannot convert {s} to bool") diff --git a/utils/activitypub_utils.py b/utils/activitypub_utils.py deleted file mode 100644 index 3204237..0000000 --- a/utils/activitypub_utils.py +++ /dev/null @@ -1,68 +0,0 @@ -from typing import Any -from typing import Dict -from typing import List -from typing import Optional - -import requests - -from .errors import RecursionLimitExceededError -from .errors import UnexpectedActivityTypeError - - -def _do_req(url: str, headers: Dict[str, str]) -> Dict[str, Any]: - resp = requests.get(url, headers=headers) - resp.raise_for_status() - return resp.json() - - -def parse_collection( - payload: Optional[Dict[str, Any]] = None, - url: Optional[str] = None, - user_agent: Optional[str] = None, - level: int = 0, - do_req: Any = _do_req, -) -> List[str]: - """Resolve/fetch a `Collection`/`OrderedCollection`.""" - if level > 3: - raise RecursionLimitExceededError('recursion limit exceeded') - - # Go through all the pages - headers = {'Accept': 'application/activity+json'} - if user_agent: - headers['User-Agent'] = user_agent - - out: List[str] = [] - if url: - payload = do_req(url, headers) - if not payload: - raise ValueError('must at least prove a payload or an URL') - - if payload['type'] in ['Collection', 'OrderedCollection']: - if 'orderedItems' in payload: - return payload['orderedItems'] - if 'items' in payload: - return payload['items'] - if 'first' in payload: - if 'orderedItems' in payload['first']: - out.extend(payload['first']['orderedItems']) - if 'items' in payload['first']: - out.extend(payload['first']['items']) - n = payload['first'].get('next') - if n: - out.extend(parse_collection(url=n, user_agent=user_agent, level=level+1, do_req=do_req)) - return out - - while payload: - if payload['type'] in ['CollectionPage', 'OrderedCollectionPage']: - if 'orderedItems' in payload: - out.extend(payload['orderedItems']) - if 'items' in payload: - out.extend(payload['items']) - n = payload.get('next') - if n is None: - break - payload = do_req(n, headers) - else: - raise UnexpectedActivityTypeError('unexpected activity type {}'.format(payload['type'])) - - return out diff --git a/utils/actor_service.py b/utils/actor_service.py index f13a6cf..bb97131 100644 --- a/utils/actor_service.py +++ b/utils/actor_service.py @@ -17,7 +17,7 @@ class NotAnActorError(Exception): class ActorService(object): def __init__(self, user_agent, col, actor_id, actor_data, instances): - logger.debug(f'Initializing ActorService user_agent={user_agent}') + logger.debug(f"Initializing ActorService user_agent={user_agent}") self._user_agent = user_agent self._col = col self._in_mem = {actor_id: actor_data} @@ -25,57 +25,70 @@ class ActorService(object): self._known_instances = set() def _fetch(self, actor_url): - logger.debug(f'fetching remote object {actor_url}') + logger.debug(f"fetching remote object {actor_url}") check_url(actor_url) - resp = requests.get(actor_url, headers={ - 'Accept': 'application/activity+json', - 'User-Agent': self._user_agent, - }) + resp = requests.get( + actor_url, + headers={ + "Accept": "application/activity+json", + "User-Agent": self._user_agent, + }, + ) if resp.status_code == 404: - raise ActivityNotFoundError(f'{actor_url} cannot be fetched, 404 not found error') + raise ActivityNotFoundError( + f"{actor_url} cannot be fetched, 404 not found error" + ) resp.raise_for_status() return resp.json() def get(self, actor_url, reload_cache=False): - logger.info(f'get actor {actor_url} (reload_cache={reload_cache})') + logger.info(f"get actor {actor_url} (reload_cache={reload_cache})") if actor_url in self._in_mem: return self._in_mem[actor_url] - instance = urlparse(actor_url)._replace(path='', query='', fragment='').geturl() + instance = urlparse(actor_url)._replace(path="", query="", fragment="").geturl() if instance not in self._known_instances: self._known_instances.add(instance) - if not self._instances.find_one({'instance': instance}): - self._instances.insert({'instance': instance, 'first_object': actor_url}) + if not self._instances.find_one({"instance": instance}): + self._instances.insert( + {"instance": instance, "first_object": actor_url} + ) if reload_cache: actor = self._fetch(actor_url) self._in_mem[actor_url] = actor - self._col.update({'actor_id': actor_url}, {'$set': {'cached_response': actor}}, upsert=True) + self._col.update( + {"actor_id": actor_url}, + {"$set": {"cached_response": actor}}, + upsert=True, + ) return actor - cached_actor = self._col.find_one({'actor_id': actor_url}) + cached_actor = self._col.find_one({"actor_id": actor_url}) if cached_actor: - return cached_actor['cached_response'] + return cached_actor["cached_response"] actor = self._fetch(actor_url) - if not 'type' in actor: + if not "type" in actor: raise NotAnActorError(None) - if actor['type'] != 'Person': + if actor["type"] != "Person": raise NotAnActorError(actor) - self._col.update({'actor_id': actor_url}, {'$set': {'cached_response': actor}}, upsert=True) + self._col.update( + {"actor_id": actor_url}, {"$set": {"cached_response": actor}}, upsert=True + ) self._in_mem[actor_url] = actor return actor def get_public_key(self, actor_url, reload_cache=False): profile = self.get(actor_url, reload_cache=reload_cache) - pub = profile['publicKey'] - return pub['id'], RSA.importKey(pub['publicKeyPem']) + pub = profile["publicKey"] + return pub["id"], RSA.importKey(pub["publicKeyPem"]) def get_inbox_url(self, actor_url, reload_cache=False): profile = self.get(actor_url, reload_cache=reload_cache) - return profile.get('inbox') + return profile.get("inbox") diff --git a/utils/content_helper.py b/utils/content_helper.py deleted file mode 100644 index 8ea8cf2..0000000 --- a/utils/content_helper.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import typing -from typing import Any -from typing import Dict -from typing import List -from typing import Optional -from typing import Tuple -from typing import Type -from typing import Union - -from bleach.linkifier import Linker -from markdown import markdown - -from config import ACTOR_SERVICE -from config import BASE_URL -from config import ID -from config import USERNAME -from utils.webfinger import get_actor_url - - -def set_attrs(attrs, new=False): - attrs[(None, u'target')] = u'_blank' - attrs[(None, u'class')] = u'external' - attrs[(None, u'rel')] = u'noopener' - attrs[(None, u'title')] = attrs[(None, u'href')] - return attrs - - -LINKER = Linker(callbacks=[set_attrs]) -HASHTAG_REGEX = re.compile(r"(#[\d\w\.]+)") -MENTION_REGEX = re.compile(r"@[\d\w_.+-]+@[\d\w-]+\.[\d\w\-.]+") - - -def hashtagify(content: str) -> Tuple[str, List[Dict[str, str]]]: - tags = [] - for hashtag in re.findall(HASHTAG_REGEX, content): - tag = hashtag[1:] - link = f'' - tags.append(dict(href=f'{BASE_URL}/tags/{tag}', name=hashtag, type='Hashtag')) - content = content.replace(hashtag, link) - return content, tags - - -def mentionify(content: str) -> Tuple[str, List[Dict[str, str]]]: - tags = [] - for mention in re.findall(MENTION_REGEX, content): - _, username, domain = mention.split('@') - actor_url = get_actor_url(mention) - p = ACTOR_SERVICE.get(actor_url) - print(p) - tags.append(dict(type='Mention', href=p['id'], name=mention)) - link = f'@{username}' - content = content.replace(mention, link) - return content, tags - - -def parse_markdown(content: str) -> Tuple[str, List[Dict[str, str]]]: - tags = [] - content = LINKER.linkify(content) - content, hashtag_tags = hashtagify(content) - tags.extend(hashtag_tags) - content, mention_tags = mentionify(content) - tags.extend(mention_tags) - content = markdown(content) - return content, tags diff --git a/utils/errors.py b/utils/errors.py deleted file mode 100644 index 7ffe744..0000000 --- a/utils/errors.py +++ /dev/null @@ -1,37 +0,0 @@ - -class Error(Exception): - status_code = 400 - - def __init__(self, message, status_code=None, payload=None): - Exception.__init__(self) - self.message = message - if status_code is not None: - self.status_code = status_code - self.payload = payload - - def to_dict(self): - rv = dict(self.payload or ()) - rv['message'] = self.message - return rv - - def __repr__(self): - return f'{self.__class__.__qualname__}({self.message!r}, payload={self.payload!r}, status_code={self.status_code})' - - -class NotFromOutboxError(Error): - pass - -class ActivityNotFoundError(Error): - status_code = 404 - - -class BadActivityError(Error): - pass - - -class RecursionLimitExceededError(BadActivityError): - pass - - -class UnexpectedActivityTypeError(BadActivityError): - pass diff --git a/utils/httpsig.py b/utils/httpsig.py deleted file mode 100644 index 609ec3d..0000000 --- a/utils/httpsig.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Implements HTTP signature for Flask requests. - -Mastodon instances won't accept requests that are not signed using this scheme. - -""" -import base64 -import hashlib -import logging -from datetime import datetime -from typing import Any -from typing import Dict -from typing import Optional -from urllib.parse import urlparse - -from Crypto.Hash import SHA256 -from Crypto.Signature import PKCS1_v1_5 -from flask import request -from requests.auth import AuthBase - -logger = logging.getLogger(__name__) - - -def _build_signed_string(signed_headers: str, method: str, path: str, headers: Any, body_digest: str) -> str: - out = [] - for signed_header in signed_headers.split(' '): - if signed_header == '(request-target)': - out.append('(request-target): '+method.lower()+' '+path) - elif signed_header == 'digest': - out.append('digest: '+body_digest) - else: - out.append(signed_header+': '+headers[signed_header]) - return '\n'.join(out) - - -def _parse_sig_header(val: Optional[str]) -> Optional[Dict[str, str]]: - if not val: - return None - out = {} - for data in val.split(','): - k, v = data.split('=', 1) - out[k] = v[1:len(v)-1] - return out - - -def _verify_h(signed_string, signature, pubkey): - signer = PKCS1_v1_5.new(pubkey) - digest = SHA256.new() - digest.update(signed_string.encode('utf-8')) - return signer.verify(digest, signature) - - -def _body_digest() -> str: - h = hashlib.new('sha256') - h.update(request.data) - return 'SHA-256='+base64.b64encode(h.digest()).decode('utf-8') - - -def verify_request(actor_service) -> bool: - hsig = _parse_sig_header(request.headers.get('Signature')) - if not hsig: - logger.debug('no signature in header') - return False - logger.debug(f'hsig={hsig}') - signed_string = _build_signed_string(hsig['headers'], request.method, request.path, request.headers, _body_digest()) - _, rk = actor_service.get_public_key(hsig['keyId']) - return _verify_h(signed_string, base64.b64decode(hsig['signature']), rk) - - -class HTTPSigAuth(AuthBase): - def __init__(self, keyid, privkey): - self.keyid = keyid - self.privkey = privkey - - def __call__(self, r): - logger.info(f'keyid={self.keyid}') - host = urlparse(r.url).netloc - bh = hashlib.new('sha256') - bh.update(r.body.encode('utf-8')) - bodydigest = 'SHA-256='+base64.b64encode(bh.digest()).decode('utf-8') - date = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT') - r.headers.update({'Digest': bodydigest, 'Date': date}) - r.headers.update({'Host': host}) - sigheaders = '(request-target) user-agent host date digest content-type' - to_be_signed = _build_signed_string(sigheaders, r.method, r.path_url, r.headers, bodydigest) - signer = PKCS1_v1_5.new(self.privkey) - digest = SHA256.new() - digest.update(to_be_signed.encode('utf-8')) - sig = base64.b64encode(signer.sign(digest)) - sig = sig.decode('utf-8') - headers = { - 'Signature': f'keyId="{self.keyid}",algorithm="rsa-sha256",headers="{sigheaders}",signature="{sig}"' - } - logger.info(f'signed request headers={headers}') - r.headers.update(headers) - return r diff --git a/utils/linked_data_sig.py b/utils/linked_data_sig.py deleted file mode 100644 index b75166e..0000000 --- a/utils/linked_data_sig.py +++ /dev/null @@ -1,69 +0,0 @@ -import base64 -import hashlib -from datetime import datetime -from typing import Any -from typing import Dict - -from Crypto.Hash import SHA256 -from Crypto.Signature import PKCS1_v1_5 -from pyld import jsonld - -# cache the downloaded "schemas", otherwise the library is super slow -# (https://github.com/digitalbazaar/pyld/issues/70) -_CACHE: Dict[str, Any] = {} -LOADER = jsonld.requests_document_loader() - -def _caching_document_loader(url: str) -> Any: - if url in _CACHE: - return _CACHE[url] - resp = LOADER(url) - _CACHE[url] = resp - return resp - -jsonld.set_document_loader(_caching_document_loader) - - -def options_hash(doc): - doc = dict(doc['signature']) - for k in ['type', 'id', 'signatureValue']: - if k in doc: - del doc[k] - doc['@context'] = 'https://w3id.org/identity/v1' - normalized = jsonld.normalize(doc, {'algorithm': 'URDNA2015', 'format': 'application/nquads'}) - h = hashlib.new('sha256') - h.update(normalized.encode('utf-8')) - return h.hexdigest() - - -def doc_hash(doc): - doc = dict(doc) - if 'signature' in doc: - del doc['signature'] - normalized = jsonld.normalize(doc, {'algorithm': 'URDNA2015', 'format': 'application/nquads'}) - h = hashlib.new('sha256') - h.update(normalized.encode('utf-8')) - return h.hexdigest() - - -def verify_signature(doc, pubkey): - to_be_signed = options_hash(doc) + doc_hash(doc) - signature = doc['signature']['signatureValue'] - signer = PKCS1_v1_5.new(pubkey) - digest = SHA256.new() - digest.update(to_be_signed.encode('utf-8')) - return signer.verify(digest, base64.b64decode(signature)) - - -def generate_signature(doc, privkey): - options = { - 'type': 'RsaSignature2017', - 'creator': doc['actor'] + '#main-key', - 'created': datetime.utcnow().replace(microsecond=0).isoformat() + 'Z', - } - doc['signature'] = options - to_be_signed = options_hash(doc) + doc_hash(doc) - signer = PKCS1_v1_5.new(privkey) - digest = SHA256.new() - digest.update(to_be_signed.encode('utf-8')) - sig = base64.b64encode(signer.sign(digest)) - options['signatureValue'] = sig.decode('utf-8') diff --git a/utils/object_service.py b/utils/object_service.py index 594fa10..8ce8d11 100644 --- a/utils/object_service.py +++ b/utils/object_service.py @@ -16,53 +16,100 @@ class ObjectService(object): self._known_instances = set() def _fetch_remote(self, object_id): - print(f'fetch remote {object_id}') + print(f"fetch remote {object_id}") check_url(object_id) - resp = requests.get(object_id, headers={ - 'Accept': 'application/activity+json', - 'User-Agent': self._user_agent, - }) + resp = requests.get( + object_id, + headers={ + "Accept": "application/activity+json", + "User-Agent": self._user_agent, + }, + ) if resp.status_code == 404: - raise ActivityNotFoundError(f'{object_id} cannot be fetched, 404 error not found') + raise ActivityNotFoundError( + f"{object_id} cannot be fetched, 404 error not found" + ) resp.raise_for_status() return resp.json() def _fetch(self, object_id): - instance = urlparse(object_id)._replace(path='', query='', fragment='').geturl() + instance = urlparse(object_id)._replace(path="", query="", fragment="").geturl() if instance not in self._known_instances: self._known_instances.add(instance) - if not self._instances.find_one({'instance': instance}): - self._instances.insert({'instance': instance, 'first_object': object_id}) + if not self._instances.find_one({"instance": instance}): + self._instances.insert( + {"instance": instance, "first_object": object_id} + ) - obj = self._inbox.find_one({'$or': [{'remote_id': object_id}, {'type': 'Create', 'activity.object.id': object_id}]}) + obj = self._inbox.find_one( + { + "$or": [ + {"remote_id": object_id}, + {"type": "Create", "activity.object.id": object_id}, + ] + } + ) if obj: - if obj['remote_id'] == object_id: - return obj['activity'] - return obj['activity']['object'] + if obj["remote_id"] == object_id: + return obj["activity"] + return obj["activity"]["object"] - obj = self._outbox.find_one({'$or': [{'remote_id': object_id}, {'type': 'Create', 'activity.object.id': object_id}]}) + obj = self._outbox.find_one( + { + "$or": [ + {"remote_id": object_id}, + {"type": "Create", "activity.object.id": object_id}, + ] + } + ) if obj: - if obj['remote_id'] == object_id: - return obj['activity'] - return obj['activity']['object'] + if obj["remote_id"] == object_id: + return obj["activity"] + return obj["activity"]["object"] return self._fetch_remote(object_id) - def get(self, object_id, reload_cache=False, part_of_stream=False, announce_published=None): + def get( + self, + object_id, + reload_cache=False, + part_of_stream=False, + announce_published=None, + ): if reload_cache: obj = self._fetch(object_id) - self._col.update({'object_id': object_id}, {'$set': {'cached_object': obj, 'meta.part_of_stream': part_of_stream, 'meta.announce_published': announce_published}}, upsert=True) + self._col.update( + {"object_id": object_id}, + { + "$set": { + "cached_object": obj, + "meta.part_of_stream": part_of_stream, + "meta.announce_published": announce_published, + } + }, + upsert=True, + ) return obj - cached_object = self._col.find_one({'object_id': object_id}) + cached_object = self._col.find_one({"object_id": object_id}) if cached_object: - print(f'ObjectService: {cached_object}') - return cached_object['cached_object'] + print(f"ObjectService: {cached_object}") + return cached_object["cached_object"] obj = self._fetch(object_id) - self._col.update({'object_id': object_id}, {'$set': {'cached_object': obj, 'meta.part_of_stream': part_of_stream, 'meta.announce_published': announce_published}}, upsert=True) + self._col.update( + {"object_id": object_id}, + { + "$set": { + "cached_object": obj, + "meta.part_of_stream": part_of_stream, + "meta.announce_published": announce_published, + } + }, + upsert=True, + ) # print(f'ObjectService: {obj}') return obj diff --git a/utils/opengraph.py b/utils/opengraph.py index 8bafece..597ad3c 100644 --- a/utils/opengraph.py +++ b/utils/opengraph.py @@ -1,37 +1,34 @@ -import ipaddress -from urllib.parse import urlparse - import opengraph import requests from bs4 import BeautifulSoup -from .urlutils import check_url -from .urlutils import is_url_valid +from little_boxes.urlutils import check_url +from little_boxes.urlutils import is_url_valid def links_from_note(note): - tags_href= set() - for t in note.get('tag', []): - h = t.get('href') + tags_href = set() + for t in note.get("tag", []): + h = t.get("href") if h: # TODO(tsileo): fetch the URL for Actor profile, type=mention tags_href.add(h) links = set() - soup = BeautifulSoup(note['content']) - for link in soup.find_all('a'): - h = link.get('href') - if h.startswith('http') and h not in tags_href and is_url_valid(h): + soup = BeautifulSoup(note["content"]) + for link in soup.find_all("a"): + h = link.get("href") + if h.startswith("http") and h not in tags_href and is_url_valid(h): links.add(h) return links def fetch_og_metadata(user_agent, col, remote_id): - doc = col.find_one({'remote_id': remote_id}) + doc = col.find_one({"remote_id": remote_id}) if not doc: raise ValueError - note = doc['activity']['object'] + note = doc["activity"]["object"] print(note) links = links_from_note(note) if not links: @@ -40,9 +37,11 @@ def fetch_og_metadata(user_agent, col, remote_id): htmls = [] for l in links: check_url(l) - r = requests.get(l, headers={'User-Agent': user_agent}) + r = requests.get(l, headers={"User-Agent": user_agent}) r.raise_for_status() htmls.append(r.text) links_og_metadata = [dict(opengraph.OpenGraph(html=html)) for html in htmls] - col.update_one({'remote_id': remote_id}, {'$set': {'meta.og_metadata': links_og_metadata}}) + col.update_one( + {"remote_id": remote_id}, {"$set": {"meta.og_metadata": links_og_metadata}} + ) return len(links) diff --git a/utils/urlutils.py b/utils/urlutils.py deleted file mode 100644 index 360d209..0000000 --- a/utils/urlutils.py +++ /dev/null @@ -1,47 +0,0 @@ -import ipaddress -import logging -import os -import socket -from urllib.parse import urlparse - -from . import strtobool -from .errors import Error - -logger = logging.getLogger(__name__) - - -class InvalidURLError(Error): - pass - - -def is_url_valid(url: str) -> bool: - parsed = urlparse(url) - if parsed.scheme not in ['http', 'https']: - return False - - # XXX in debug mode, we want to allow requests to localhost to test the federation with local instances - debug_mode = strtobool(os.getenv('MICROBLOGPUB_DEBUG', 'false')) - if debug_mode: - return True - - if parsed.hostname in ['localhost']: - return False - - try: - ip_address = socket.getaddrinfo(parsed.hostname, parsed.port or 80)[0][4][0] - except socket.gaierror: - logger.exception(f'failed to lookup url {url}') - return False - - if ipaddress.ip_address(ip_address).is_private: - logger.info(f'rejecting private URL {url}') - return False - - return True - - -def check_url(url: str) -> None: - if not is_url_valid(url): - raise InvalidURLError(f'"{url}" is invalid') - - return None diff --git a/utils/webfinger.py b/utils/webfinger.py deleted file mode 100644 index 344dc01..0000000 --- a/utils/webfinger.py +++ /dev/null @@ -1,75 +0,0 @@ -import logging -from typing import Any -from typing import Dict -from typing import Optional -from urllib.parse import urlparse - -import requests - -from .urlutils import check_url - -logger = logging.getLogger(__name__) - - -def webfinger(resource: str) -> Optional[Dict[str, Any]]: - """Mastodon-like WebFinger resolution to retrieve the activity stream Actor URL. - """ - logger.info(f'performing webfinger resolution for {resource}') - protos = ['https', 'http'] - if resource.startswith('http://'): - protos.reverse() - host = urlparse(resource).netloc - elif resource.startswith('https://'): - host = urlparse(resource).netloc - else: - if resource.startswith('acct:'): - resource = resource[5:] - if resource.startswith('@'): - resource = resource[1:] - _, host = resource.split('@', 1) - resource='acct:'+resource - - # Security check on the url (like not calling localhost) - check_url(f'https://{host}') - - for i, proto in enumerate(protos): - try: - url = f'{proto}://{host}/.well-known/webfinger' - resp = requests.get( - url, - {'resource': resource} - ) - except requests.ConnectionError: - # If we tried https first and the domain is "http only" - if i == 0: - continue - break - if resp.status_code == 404: - return None - resp.raise_for_status() - return resp.json() - - -def get_remote_follow_template(resource: str) -> Optional[str]: - data = webfinger(resource) - if data is None: - return None - for link in data['links']: - if link.get('rel') == 'http://ostatus.org/schema/1.0/subscribe': - return link.get('template') - return None - - -def get_actor_url(resource: str) -> Optional[str]: - """Mastodon-like WebFinger resolution to retrieve the activity stream Actor URL. - - Returns: - the Actor URL or None if the resolution failed. - """ - data = webfinger(resource) - if data is None: - return None - for link in data['links']: - if link.get('rel') == 'self' and link.get('type') == 'application/activity+json': - return link.get('href') - return None From 7f65fdfc907f4c45ae726f82b46acbfd8d1778be Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 17 Jun 2018 20:51:23 +0200 Subject: [PATCH 0115/1425] Switched little_boxes --- activitypub.py | 30 ++++++++-- app.py | 13 +++-- config.py | 4 +- utils/actor_service.py | 94 -------------------------------- utils/object_service.py | 118 ++++------------------------------------ 5 files changed, 48 insertions(+), 211 deletions(-) delete mode 100644 utils/actor_service.py diff --git a/activitypub.py b/activitypub.py index 0ff1e7a..8dd4399 100644 --- a/activitypub.py +++ b/activitypub.py @@ -21,11 +21,10 @@ from little_boxes import activitypub as ap from little_boxes.backend import Backend from little_boxes.collection import parse_collection as ap_parse_collection from little_boxes.errors import Error +from little_boxes.errors import ActivityNotFoundError logger = logging.getLogger(__name__) -MY_PERSON = ap.Person(**ME) - def _remove_id(doc: ap.ObjectType) -> ap.ObjectType: """Helper for removing MongoDB's `_id` field.""" @@ -46,7 +45,7 @@ def ensure_it_is_me(f): """Method decorator used to track the events fired during tests.""" def wrapper(*args, **kwargs): - if args[1].id != MY_PERSON.id: + if args[1].id != ME["id"]: raise Error("unexpected actor") return f(*args, **kwargs) @@ -87,7 +86,22 @@ class MicroblogPubBackend(Backend): ) def fetch_iri(self, iri: str) -> ap.ObjectType: - # FIXME(tsileo): implements caching + if iri == ME["id"]: + return ME + + # Check if the activity is owned by this server + if iri.startswith(BASE_URL): + data = DB.outbox.find_one({"remote_id": iri}) + if not data: + raise ActivityNotFoundError(f"{iri} not found on this server") + return data["activity"] + + # Check if the activity is stored in the inbox + data = DB.inbox.find_one({"remote_id": iri}) + if data: + return data["activity"] + + # Fetch the URL via HTTP return super().fetch_iri(iri) @ensure_it_is_me @@ -149,7 +163,7 @@ class MicroblogPubBackend(Backend): ) @ensure_it_is_me - def outobx_like(self, as_actor: ap.Person, like: ap.Like) -> None: + def outbox_like(self, as_actor: ap.Person, like: ap.Like) -> None: obj = like.get_object() # Unlikely, but an actor can like it's own post DB.outbox.update_one( @@ -273,6 +287,12 @@ class MicroblogPubBackend(Backend): # FIXME(tsileo): should send an Update (but not a partial one, to all the note's recipients # (create a new Update with the result of the update, and send it without saving it?) + def outbox_create(self, as_actor: ap.Person, create: ap.Create) -> None: + pass + + def inbox_create(self, as_actor: ap.Person, create: ap.Create) -> None: + pass + def gen_feed(): fg = FeedGenerator() diff --git a/app.py b/app.py index 33374a2..0ab5e29 100644 --- a/app.py +++ b/app.py @@ -36,9 +36,7 @@ from werkzeug.utils import secure_filename import activitypub import config -from activitypub import MY_PERSON from activitypub import embed_collection -from config import ACTOR_SERVICE from config import ADMIN_API_KEY from config import BASE_URL from config import DB @@ -49,7 +47,6 @@ from config import ID from config import JWT from config import KEY from config import ME -from config import OBJECT_SERVICE from config import PASS from config import USERNAME from config import VERSION @@ -63,10 +60,18 @@ from little_boxes.errors import ActivityNotFoundError from little_boxes.errors import Error from little_boxes.errors import NotFromOutboxError from little_boxes.httpsig import HTTPSigAuth -from little_boxes.httpsig import verify_request +# from little_boxes.httpsig import verify_request from little_boxes.webfinger import get_actor_url from little_boxes.webfinger import get_remote_follow_template from utils.key import get_secret_key +from utils.object_service import ObjectService + +OBJECT_SERVICE = ACTOR_SERVICE = ObjectService() + +back = activitypub.MicroblogPubBackend() +ap.use_backend(back) + +MY_PERSON = ap.Person(**ME) app = Flask(__name__) app.secret_key = get_secret_key("flask") diff --git a/config.py b/config.py index 8478186..44bca9c 100644 --- a/config.py +++ b/config.py @@ -84,9 +84,9 @@ JWT = JSONWebSignatureSerializer(JWT_SECRET) def _admin_jwt_token() -> str: - return JWT.dumps({"me": "ADMIN", "ts": datetime.now().timestamp()}).decode( + return JWT.dumps({"me": "ADMIN", "ts": datetime.now().timestamp()}).decode( # type: ignore "utf-8" - ) # type: ignore + ) ADMIN_API_KEY = get_secret_key("admin_api_key", _admin_jwt_token) diff --git a/utils/actor_service.py b/utils/actor_service.py deleted file mode 100644 index bb97131..0000000 --- a/utils/actor_service.py +++ /dev/null @@ -1,94 +0,0 @@ -import logging -from urllib.parse import urlparse - -import requests -from Crypto.PublicKey import RSA - -from .errors import ActivityNotFoundError -from .urlutils import check_url - -logger = logging.getLogger(__name__) - - -class NotAnActorError(Exception): - def __init__(self, activity): - self.activity = activity - - -class ActorService(object): - def __init__(self, user_agent, col, actor_id, actor_data, instances): - logger.debug(f"Initializing ActorService user_agent={user_agent}") - self._user_agent = user_agent - self._col = col - self._in_mem = {actor_id: actor_data} - self._instances = instances - self._known_instances = set() - - def _fetch(self, actor_url): - logger.debug(f"fetching remote object {actor_url}") - - check_url(actor_url) - - resp = requests.get( - actor_url, - headers={ - "Accept": "application/activity+json", - "User-Agent": self._user_agent, - }, - ) - if resp.status_code == 404: - raise ActivityNotFoundError( - f"{actor_url} cannot be fetched, 404 not found error" - ) - - resp.raise_for_status() - return resp.json() - - def get(self, actor_url, reload_cache=False): - logger.info(f"get actor {actor_url} (reload_cache={reload_cache})") - - if actor_url in self._in_mem: - return self._in_mem[actor_url] - - instance = urlparse(actor_url)._replace(path="", query="", fragment="").geturl() - if instance not in self._known_instances: - self._known_instances.add(instance) - if not self._instances.find_one({"instance": instance}): - self._instances.insert( - {"instance": instance, "first_object": actor_url} - ) - - if reload_cache: - actor = self._fetch(actor_url) - self._in_mem[actor_url] = actor - self._col.update( - {"actor_id": actor_url}, - {"$set": {"cached_response": actor}}, - upsert=True, - ) - return actor - - cached_actor = self._col.find_one({"actor_id": actor_url}) - if cached_actor: - return cached_actor["cached_response"] - - actor = self._fetch(actor_url) - if not "type" in actor: - raise NotAnActorError(None) - if actor["type"] != "Person": - raise NotAnActorError(actor) - - self._col.update( - {"actor_id": actor_url}, {"$set": {"cached_response": actor}}, upsert=True - ) - self._in_mem[actor_url] = actor - return actor - - def get_public_key(self, actor_url, reload_cache=False): - profile = self.get(actor_url, reload_cache=reload_cache) - pub = profile["publicKey"] - return pub["id"], RSA.importKey(pub["publicKeyPem"]) - - def get_inbox_url(self, actor_url, reload_cache=False): - profile = self.get(actor_url, reload_cache=reload_cache) - return profile.get("inbox") diff --git a/utils/object_service.py b/utils/object_service.py index 8ce8d11..e46f9b1 100644 --- a/utils/object_service.py +++ b/utils/object_service.py @@ -1,115 +1,21 @@ -from urllib.parse import urlparse +import logging -import requests +from little_boxes.activitypub import get_backend -from .errors import ActivityNotFoundError -from .urlutils import check_url +logger = logging.getLogger(__name__) class ObjectService(object): - def __init__(self, user_agent, col, inbox, outbox, instances): - self._user_agent = user_agent - self._col = col - self._inbox = inbox - self._outbox = outbox - self._instances = instances - self._known_instances = set() + def __init__(self): + logger.debug("Initializing ObjectService") + self._cache = {} - def _fetch_remote(self, object_id): - print(f"fetch remote {object_id}") - check_url(object_id) - resp = requests.get( - object_id, - headers={ - "Accept": "application/activity+json", - "User-Agent": self._user_agent, - }, - ) - if resp.status_code == 404: - raise ActivityNotFoundError( - f"{object_id} cannot be fetched, 404 error not found" - ) + def get(self, iri, reload_cache=False): + logger.info(f"get actor {iri} (reload_cache={reload_cache})") - resp.raise_for_status() - return resp.json() - - def _fetch(self, object_id): - instance = urlparse(object_id)._replace(path="", query="", fragment="").geturl() - if instance not in self._known_instances: - self._known_instances.add(instance) - if not self._instances.find_one({"instance": instance}): - self._instances.insert( - {"instance": instance, "first_object": object_id} - ) - - obj = self._inbox.find_one( - { - "$or": [ - {"remote_id": object_id}, - {"type": "Create", "activity.object.id": object_id}, - ] - } - ) - if obj: - if obj["remote_id"] == object_id: - return obj["activity"] - return obj["activity"]["object"] - - obj = self._outbox.find_one( - { - "$or": [ - {"remote_id": object_id}, - {"type": "Create", "activity.object.id": object_id}, - ] - } - ) - if obj: - if obj["remote_id"] == object_id: - return obj["activity"] - return obj["activity"]["object"] - - return self._fetch_remote(object_id) - - def get( - self, - object_id, - reload_cache=False, - part_of_stream=False, - announce_published=None, - ): - if reload_cache: - obj = self._fetch(object_id) - self._col.update( - {"object_id": object_id}, - { - "$set": { - "cached_object": obj, - "meta.part_of_stream": part_of_stream, - "meta.announce_published": announce_published, - } - }, - upsert=True, - ) - return obj - - cached_object = self._col.find_one({"object_id": object_id}) - if cached_object: - print(f"ObjectService: {cached_object}") - return cached_object["cached_object"] - - obj = self._fetch(object_id) - - self._col.update( - {"object_id": object_id}, - { - "$set": { - "cached_object": obj, - "meta.part_of_stream": part_of_stream, - "meta.announce_published": announce_published, - } - }, - upsert=True, - ) - # print(f'ObjectService: {obj}') + if not reload_cache and iri in self._cache: + return self._cache[iri] + obj = get_backend().fetch_iri(iri) + self._cache[iri] = obj return obj From 2c53573b7e081c57d4b92da7a1a786bd41621bbe Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 17 Jun 2018 20:58:50 +0200 Subject: [PATCH 0116/1425] Fix the tests --- tests/federation_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/federation_test.py b/tests/federation_test.py index 1bdda92..8f6ccac 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -6,7 +6,7 @@ from typing import Tuple import requests from html2text import html2text -from utils import activitypub_utils +from little_boxes.collection import parse_collection def resp2plaintext(resp): @@ -33,14 +33,14 @@ class Instance(object): def _do_req(self, url, headers): """Used to parse collection.""" url = url.replace(self.docker_url, self.host_url) - resp = requests.get(url, headers=headers) + resp = requests.get(url, headers={'Accept': 'application/actiivty+json'}) resp.raise_for_status() return resp.json() def _parse_collection(self, payload=None, url=None): """Parses a collection (go through all the pages).""" - return activitypub_utils.parse_collection( - url=url, payload=payload, do_req=self._do_req + return parse_collection( + url=url, payload=payload, fetcher=self._do_req, ) def ping(self): From 781ed8efe265f5057a009f6d4b3385bd84810c00 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 17 Jun 2018 21:54:16 +0200 Subject: [PATCH 0117/1425] Some tests are passing --- activitypub.py | 19 +++++++++---------- app.py | 10 +++++----- dev-requirements.txt | 1 + 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/activitypub.py b/activitypub.py index 8dd4399..d7e55a1 100644 --- a/activitypub.py +++ b/activitypub.py @@ -79,7 +79,7 @@ class MicroblogPubBackend(Backend): DB.outbox.find_one( { "type": ap.ActivityType.BLOCK.value, - "activity.object": as_actor.id, + "activity.object": actor_id, "meta.undo": False, } ) @@ -92,14 +92,13 @@ class MicroblogPubBackend(Backend): # Check if the activity is owned by this server if iri.startswith(BASE_URL): data = DB.outbox.find_one({"remote_id": iri}) - if not data: - raise ActivityNotFoundError(f"{iri} not found on this server") - return data["activity"] - - # Check if the activity is stored in the inbox - data = DB.inbox.find_one({"remote_id": iri}) - if data: - return data["activity"] + if data: + return data["activity"] + else: + # Check if the activity is stored in the inbox + data = DB.inbox.find_one({"remote_id": iri}) + if data: + return data["activity"] # Fetch the URL via HTTP return super().fetch_iri(iri) @@ -142,7 +141,7 @@ class MicroblogPubBackend(Backend): @ensure_it_is_me def new_following(self, as_actor: ap.Person, follow: ap.Follow) -> None: - remote_actor = follow.get_actor().id + remote_actor = follow.get_object().id if DB.following.find({"remote_actor": remote_actor}).count() == 0: DB.following.insert_one({"remote_actor": remote_actor}) diff --git a/app.py b/app.py index 0ab5e29..0e3461d 100644 --- a/app.py +++ b/app.py @@ -384,7 +384,7 @@ def authorize_follow(): if DB.following.find({"remote_actor": actor}).count() > 0: return redirect("/following") - follow = activitypub.Follow(actor=MY_PERSON, object=actor) + follow = activitypub.Follow(actor=MY_PERSON.id, object=actor) OUTBOX.post(follow) return redirect("/following") @@ -1182,7 +1182,7 @@ def api_upload(): content = request.args.get("content") to = request.args.get("to") note = ap.Note( - actor=MY_PERSON, + attributedTo=MY_PERSON.id, cc=[ID + "/followers"], to=[to if to else ap.AS_PUBLIC], content=content, # TODO(tsileo): handle markdown @@ -1225,7 +1225,7 @@ def api_new_note(): cc.append(tag["href"]) note = ap.Note( - actor=MY_PERSON, + attributedTo=MY_PERSON.id, cc=list(set(cc)), to=[to if to else ap.AS_PUBLIC], content=content, @@ -1261,7 +1261,7 @@ def api_block(): if existing: return _user_api_response(activity=existing["activity"]["id"]) - block = ap.Block(actor=MY_PERSON, object=actor) + block = ap.Block(actor=MY_PERSON.id, object=actor) OUTBOX.post(block) return _user_api_response(activity=block.id) @@ -1276,7 +1276,7 @@ def api_follow(): if existing: return _user_api_response(activity=existing["activity"]["id"]) - follow = ap.Follow(actor=MY_PERSON, object=actor) + follow = ap.Follow(actor=MY_PERSON.id, object=actor) OUTBOX.post(follow) return _user_api_response(activity=follow.id) diff --git a/dev-requirements.txt b/dev-requirements.txt index a4ab4e5..7db7fab 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,4 @@ +git+https://github.com/tsileo/little-boxes.git pytest requests html2text From 986edbd35e8491124e19614d4329f68e11bbddc8 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 17 Jun 2018 22:05:38 +0200 Subject: [PATCH 0118/1425] Fix fetch_iri --- activitypub.py | 7 ++++++- app.py | 1 + config.py | 4 +++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/activitypub.py b/activitypub.py index d7e55a1..bf09f64 100644 --- a/activitypub.py +++ b/activitypub.py @@ -21,7 +21,6 @@ from little_boxes import activitypub as ap from little_boxes.backend import Backend from little_boxes.collection import parse_collection as ap_parse_collection from little_boxes.errors import Error -from little_boxes.errors import ActivityNotFoundError logger = logging.getLogger(__name__) @@ -91,8 +90,14 @@ class MicroblogPubBackend(Backend): # Check if the activity is owned by this server if iri.startswith(BASE_URL): + is_a_note = False + if iri.endswith('/activity'): + iri = iri.replace('/activity', '') + is_a_note = True data = DB.outbox.find_one({"remote_id": iri}) if data: + if is_a_note: + return data['activity']['object'] return data["activity"] else: # Check if the activity is stored in the inbox diff --git a/app.py b/app.py index 0e3461d..1f7f0af 100644 --- a/app.py +++ b/app.py @@ -60,6 +60,7 @@ from little_boxes.errors import ActivityNotFoundError from little_boxes.errors import Error from little_boxes.errors import NotFromOutboxError from little_boxes.httpsig import HTTPSigAuth + # from little_boxes.httpsig import verify_request from little_boxes.webfinger import get_actor_url from little_boxes.webfinger import get_remote_follow_template diff --git a/config.py b/config.py index 44bca9c..f552f63 100644 --- a/config.py +++ b/config.py @@ -84,7 +84,9 @@ JWT = JSONWebSignatureSerializer(JWT_SECRET) def _admin_jwt_token() -> str: - return JWT.dumps({"me": "ADMIN", "ts": datetime.now().timestamp()}).decode( # type: ignore + return JWT.dumps( + {"me": "ADMIN", "ts": datetime.now().timestamp()} + ).decode( # type: ignore "utf-8" ) From 1338d99994e729ceb57f4a2ac6fe918968e01928 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 18 Jun 2018 22:01:21 +0200 Subject: [PATCH 0119/1425] More bugfixes and re-handle replies --- activitypub.py | 47 ++++++++++++++++++++++++++++++++---- app.py | 51 ++++++++++++++++++++++++++++------------ tests/federation_test.py | 4 ++-- 3 files changed, 80 insertions(+), 22 deletions(-) diff --git a/activitypub.py b/activitypub.py index bf09f64..b66171e 100644 --- a/activitypub.py +++ b/activitypub.py @@ -91,13 +91,13 @@ class MicroblogPubBackend(Backend): # Check if the activity is owned by this server if iri.startswith(BASE_URL): is_a_note = False - if iri.endswith('/activity'): - iri = iri.replace('/activity', '') + if iri.endswith("/activity"): + iri = iri.replace("/activity", "") is_a_note = True data = DB.outbox.find_one({"remote_id": iri}) if data: if is_a_note: - return data['activity']['object'] + return data["activity"]["object"] return data["activity"] else: # Check if the activity is stored in the inbox @@ -239,6 +239,11 @@ class MicroblogPubBackend(Backend): {"activity.object.id": delete.get_object().id}, {"$set": {"meta.deleted": True}}, ) + obj = delete.get_object() + if obj.ACTIVITY_TYPE != ActivityType.NOTE: + obj = self.fetch_iri(delete.get_object().id) + self._handle_replies_delete(as_actor, obj) + # FIXME(tsileo): handle threads # obj = delete._get_actual_object() # if obj.type_enum == ActivityType.NOTE: @@ -291,11 +296,43 @@ class MicroblogPubBackend(Backend): # FIXME(tsileo): should send an Update (but not a partial one, to all the note's recipients # (create a new Update with the result of the update, and send it without saving it?) + @ensure_it_is_me def outbox_create(self, as_actor: ap.Person, create: ap.Create) -> None: - pass + self._handle_replies(as_actor, create) + @ensure_it_is_me def inbox_create(self, as_actor: ap.Person, create: ap.Create) -> None: - pass + self._handle_replies(as_actor, create) + + @ensure_it_is_me + def _handle_replies_delete(self, as_actor: ap.Person, note: ap.Create) -> None: + in_reply_to = note.inReplyTo + if not in_reply_to: + pass + + if not DB.inbox.find_one_and_update( + {"activity.object.id": in_reply_to}, + {"$inc": {"meta.count_reply": -1, "meta.count_direct_reply": -1}}, + ): + DB.outbox.update_one( + {"activity.object.id": in_reply_to}, + {"$inc": {"meta.count_reply": -1, "meta.count_direct_reply": -1}}, + ) + + @ensure_it_is_me + def _handle_replies(self, as_actor: ap.Person, create: ap.Create) -> None: + in_reply_to = create.get_object().inReplyTo + if not in_reply_to: + pass + + if not DB.inbox.find_one_and_update( + {"activity.object.id": in_reply_to}, + {"$inc": {"meta.count_reply": 1, "meta.count_direct_reply": 1}}, + ): + DB.outbox.update_one( + {"activity.object.id": in_reply_to}, + {"$inc": {"meta.count_reply": 1, "meta.count_direct_reply": 1}}, + ) def gen_feed(): diff --git a/app.py b/app.py index 1f7f0af..fd564c1 100644 --- a/app.py +++ b/app.py @@ -553,7 +553,7 @@ def _build_thread(data, include_children=True): @app.route("/note/") def note_by_id(note_id): - data = DB.outbox.find_one({"id": note_id}) + data = DB.outbox.find_one({"remote_id": back.activity_url(note_id)}) if not data: abort(404) if data["meta"].get("deleted", False): @@ -671,17 +671,15 @@ def add_extra_collection(raw_doc: Dict[str, Any]) -> Dict[str, Any]: raw_doc["activity"]["object"]["replies"] = embed_collection( raw_doc.get("meta", {}).get("count_direct_reply", 0), - f'{ID}/outbox/{raw_doc["id"]}/replies', + f'{raw_doc["remote_id"]}/replies', ) raw_doc["activity"]["object"]["likes"] = embed_collection( - raw_doc.get("meta", {}).get("count_like", 0), - f'{ID}/outbox/{raw_doc["id"]}/likes', + raw_doc.get("meta", {}).get("count_like", 0), f'{raw_doc["remote_id"]}/likes' ) raw_doc["activity"]["object"]["shares"] = embed_collection( - raw_doc.get("meta", {}).get("count_boost", 0), - f'{ID}/outbox/{raw_doc["id"]}/shares', + raw_doc.get("meta", {}).get("count_boost", 0), f'{raw_doc["remote_id"]}/shares' ) return raw_doc @@ -740,7 +738,7 @@ def outbox(): @app.route("/outbox/") def outbox_detail(item_id): - doc = DB.outbox.find_one({"id": item_id}) + doc = DB.outbox.find_one({"remote_id": back.activity_url(item_id)}) if doc["meta"].get("deleted", False): obj = ap.parse_activity(doc["activity"]) resp = jsonify(**obj.get_object().get_tombstone()) @@ -752,7 +750,9 @@ def outbox_detail(item_id): @app.route("/outbox//activity") def outbox_activity(item_id): # TODO(tsileo): handle Tombstone - data = DB.outbox.find_one({"id": item_id, "meta.deleted": False}) + data = DB.outbox.find_one( + {"remote_id": back.activity_url(item_id), "meta.deleted": False} + ) if not data: abort(404) obj = activity_from_doc(data) @@ -766,7 +766,9 @@ def outbox_activity_replies(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one({"id": item_id, "meta.deleted": False}) + data = DB.outbox.find_one( + {"remote_id": back.activity_url(item_id), "meta.deleted": False} + ) if not data: abort(404) obj = ap.parse_activity(data["activity"]) @@ -796,7 +798,9 @@ def outbox_activity_likes(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one({"id": item_id, "meta.deleted": False}) + data = DB.outbox.find_one( + {"remote_id": back.activity_url(item_id), "meta.deleted": False} + ) if not data: abort(404) obj = ap.parse_activity(data["activity"]) @@ -829,7 +833,9 @@ def outbox_activity_shares(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one({"id": item_id, "meta.deleted": False}) + data = DB.outbox.find_one( + {"remote_id": back.activity_url(item_id), "meta.deleted": False} + ) if not data: abort(404) obj = ap.parse_activity(data["activity"]) @@ -1007,7 +1013,7 @@ def api_delete(): def api_boost(): note = _user_api_get_note() - announce = note.build_announce() + announce = note.build_announce(MY_PERSON) OUTBOX.post(announce) return _user_api_response(activity=announce.id) @@ -1018,7 +1024,7 @@ def api_boost(): def api_like(): note = _user_api_get_note() - like = note.build_like() + like = note.build_like(MY_PERSON) OUTBOX.post(like) return _user_api_response(activity=like.id) @@ -1028,7 +1034,9 @@ def api_like(): @api_required def api_undo(): oid = _user_api_arg("id") - doc = DB.outbox.find_one({"$or": [{"id": oid}, {"remote_id": oid}]}) + doc = DB.outbox.find_one( + {"$or": [{"remote_id": back.activity_url(oid)}, {"remote_id": oid}]} + ) if not doc: raise ActivityNotFoundError(f"cannot found {oid}") @@ -1141,6 +1149,15 @@ def inbox(): return Response(status=201) +def without_id(l): + out = [] + for d in l: + if "_id" in d: + del d["_id"] + out.append(d) + return out + + @app.route("/api/debug", methods=["GET", "DELETE"]) @api_required def api_debug(): @@ -1152,7 +1169,11 @@ def api_debug(): _drop_db() return flask_jsonify(message="DB dropped") - return flask_jsonify(inbox=DB.inbox.count(), outbox=DB.outbox.count()) + return flask_jsonify( + inbox=DB.inbox.count(), + outbox=DB.outbox.count(), + outbox_data=without_id(DB.outbox.find()), + ) @app.route("/api/upload", methods=["POST"]) diff --git a/tests/federation_test.py b/tests/federation_test.py index 8f6ccac..6e0a7ea 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -30,10 +30,10 @@ class Instance(object): api_key = f.read() self._auth_headers = {"Authorization": f"Bearer {api_key}"} - def _do_req(self, url, headers): + def _do_req(self, url): """Used to parse collection.""" url = url.replace(self.docker_url, self.host_url) - resp = requests.get(url, headers={'Accept': 'application/actiivty+json'}) + resp = requests.get(url, headers={'Accept': 'application/activity+json'}) resp.raise_for_status() return resp.json() From ef7a887146d683f931c0d67e3fe818047293e00f Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 18 Jun 2018 22:04:24 +0200 Subject: [PATCH 0120/1425] Bugfixes --- activitypub.py | 2 +- config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/activitypub.py b/activitypub.py index b66171e..b2f587c 100644 --- a/activitypub.py +++ b/activitypub.py @@ -240,7 +240,7 @@ class MicroblogPubBackend(Backend): {"$set": {"meta.deleted": True}}, ) obj = delete.get_object() - if obj.ACTIVITY_TYPE != ActivityType.NOTE: + if obj.ACTIVITY_TYPE != ap.ActivityType.NOTE: obj = self.fetch_iri(delete.get_object().id) self._handle_replies_delete(as_actor, obj) diff --git a/config.py b/config.py index f552f63..44659c1 100644 --- a/config.py +++ b/config.py @@ -84,7 +84,7 @@ JWT = JSONWebSignatureSerializer(JWT_SECRET) def _admin_jwt_token() -> str: - return JWT.dumps( + return JWT.dumps( # type: ignore {"me": "ADMIN", "ts": datetime.now().timestamp()} ).decode( # type: ignore "utf-8" From c3832592965bdd6f989f2b231d55ab80827a4be1 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 18 Jun 2018 23:34:07 +0200 Subject: [PATCH 0121/1425] Bugfixes --- activitypub.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/activitypub.py b/activitypub.py index b2f587c..7a929c4 100644 --- a/activitypub.py +++ b/activitypub.py @@ -241,8 +241,16 @@ class MicroblogPubBackend(Backend): ) obj = delete.get_object() if obj.ACTIVITY_TYPE != ap.ActivityType.NOTE: - obj = self.fetch_iri(delete.get_object().id) - self._handle_replies_delete(as_actor, obj) + obj = DB.inbox.find_one( + { + "activity.object.id": delete.get_object().id, + "type": ap.ActivityType.CREATE.value, + } + ) + + logger.info(f"inbox_delete handle_replies obj={obj!r}") + if obj: + self._handle_replies_delete(as_actor, obj) # FIXME(tsileo): handle threads # obj = delete._get_actual_object() @@ -257,6 +265,18 @@ class MicroblogPubBackend(Backend): {"activity.object.id": delete.get_object().id}, {"$set": {"meta.deleted": True}}, ) + obj = delete.get_object() + if delete.get_object().ACTIVITY_TYPE != ap.ActivityType.NOTE: + obj = ap.parse_activity( + DB.outbox.find_one( + { + "activity.object.id": delete.get_object().id, + "type": ap.ActivityType.CREATE.value, + } + ) + ).get_object() + + self._handle_replies_delete(as_actor, obj) @ensure_it_is_me def inbox_update(self, as_actor: ap.Person, update: ap.Update) -> None: @@ -305,7 +325,7 @@ class MicroblogPubBackend(Backend): self._handle_replies(as_actor, create) @ensure_it_is_me - def _handle_replies_delete(self, as_actor: ap.Person, note: ap.Create) -> None: + def _handle_replies_delete(self, as_actor: ap.Person, note: ap.Note) -> None: in_reply_to = note.inReplyTo if not in_reply_to: pass From a8e9b5498adb34c8bacd4f794c6cd5778a18a8e5 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 18 Jun 2018 23:57:53 +0200 Subject: [PATCH 0122/1425] Fix Delete side effects --- activitypub.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/activitypub.py b/activitypub.py index 7a929c4..21edb4c 100644 --- a/activitypub.py +++ b/activitypub.py @@ -241,12 +241,14 @@ class MicroblogPubBackend(Backend): ) obj = delete.get_object() if obj.ACTIVITY_TYPE != ap.ActivityType.NOTE: - obj = DB.inbox.find_one( - { - "activity.object.id": delete.get_object().id, - "type": ap.ActivityType.CREATE.value, - } - ) + obj = ap.parse_activity( + DB.inbox.find_one( + { + "activity.object.id": delete.get_object().id, + "type": ap.ActivityType.CREATE.value, + } + )["activity"] + ).get_object() logger.info(f"inbox_delete handle_replies obj={obj!r}") if obj: @@ -273,7 +275,7 @@ class MicroblogPubBackend(Backend): "activity.object.id": delete.get_object().id, "type": ap.ActivityType.CREATE.value, } - ) + )["activity"] ).get_object() self._handle_replies_delete(as_actor, obj) From 8d5f4a8e9825345d05ebf13f6447f70718d3a7b2 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 19 Jun 2018 00:10:19 +0200 Subject: [PATCH 0123/1425] Switch to Little boxes, fixes #1 (#8) --- activitypub.py | 1423 +++++++++-------------------------- app.py | 1459 ++++++++++++++++++++---------------- config.py | 117 ++- dev-requirements.txt | 2 + requirements.txt | 6 +- tasks.py | 63 +- tests/federation_test.py | 401 +++++----- tests/integration_test.py | 9 +- utils/__init__.py | 6 +- utils/activitypub_utils.py | 65 -- utils/actor_service.py | 81 -- utils/content_helper.py | 58 -- utils/errors.py | 37 - utils/httpsig.py | 94 --- utils/key.py | 54 +- utils/linked_data_sig.py | 70 -- utils/object_service.py | 72 +- utils/opengraph.py | 30 +- utils/urlutils.py | 47 -- utils/webfinger.py | 75 -- 20 files changed, 1529 insertions(+), 2640 deletions(-) delete mode 100644 utils/activitypub_utils.py delete mode 100644 utils/actor_service.py delete mode 100644 utils/content_helper.py delete mode 100644 utils/errors.py delete mode 100644 utils/httpsig.py delete mode 100644 utils/linked_data_sig.py delete mode 100644 utils/urlutils.py delete mode 100644 utils/webfinger.py diff --git a/activitypub.py b/activitypub.py index cdb0bd1..21edb4c 100644 --- a/activitypub.py +++ b/activitypub.py @@ -1,77 +1,35 @@ import logging -import json -import binascii -import os from datetime import datetime -from enum import Enum +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Union from bson.objectid import ObjectId -from html2text import html2text from feedgen.feed import FeedGenerator +from html2text import html2text -from utils.actor_service import NotAnActorError -from utils.errors import BadActivityError -from utils.errors import UnexpectedActivityTypeError -from utils.errors import NotFromOutboxError -from utils import activitypub_utils -from config import USERNAME, BASE_URL, ID -from config import CTX_AS, CTX_SECURITY, AS_PUBLIC -from config import DB, ME, ACTOR_SERVICE -from config import OBJECT_SERVICE -from config import PUBLIC_INSTANCES import tasks - -from typing import List, Optional, Dict, Any, Union +from config import BASE_URL +from config import DB +from config import ID +from config import ME +from config import USER_AGENT +from config import USERNAME +from little_boxes import activitypub as ap +from little_boxes.backend import Backend +from little_boxes.collection import parse_collection as ap_parse_collection +from little_boxes.errors import Error logger = logging.getLogger(__name__) -# Helper/shortcut for typing -ObjectType = Dict[str, Any] -ObjectOrIDType = Union[str, ObjectType] - -COLLECTION_CTX = [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - { - "Hashtag": "as:Hashtag", - "sensitive": "as:sensitive", - } -] - - -class ActivityType(Enum): - """Supported activity `type`.""" - ANNOUNCE = 'Announce' - BLOCK = 'Block' - LIKE = 'Like' - CREATE = 'Create' - UPDATE = 'Update' - PERSON = 'Person' - ORDERED_COLLECTION = 'OrderedCollection' - ORDERED_COLLECTION_PAGE = 'OrderedCollectionPage' - COLLECTION_PAGE = 'CollectionPage' - COLLECTION = 'Collection' - NOTE = 'Note' - ACCEPT = 'Accept' - REJECT = 'Reject' - FOLLOW = 'Follow' - DELETE = 'Delete' - UNDO = 'Undo' - IMAGE = 'Image' - TOMBSTONE = 'Tombstone' - - -def random_object_id() -> str: - """Generates a random object ID.""" - return binascii.hexlify(os.urandom(8)).decode('utf-8') - - -def _remove_id(doc: ObjectType) -> ObjectType: +def _remove_id(doc: ap.ObjectType) -> ap.ObjectType: """Helper for removing MongoDB's `_id` field.""" doc = doc.copy() - if '_id' in doc: - del(doc['_id']) + if "_id" in doc: + del (doc["_id"]) return doc @@ -82,1125 +40,458 @@ def _to_list(data: Union[List[Any], Any]) -> List[Any]: return [data] -def clean_activity(activity: ObjectType) -> Dict[str, Any]: - """Clean the activity before rendering it. - - Remove the hidden bco and bcc field - """ - for field in ['bto', 'bcc']: - if field in activity: - del(activity[field]) - if activity['type'] == 'Create' and field in activity['object']: - del(activity['object'][field]) - return activity +def ensure_it_is_me(f): + """Method decorator used to track the events fired during tests.""" + def wrapper(*args, **kwargs): + if args[1].id != ME["id"]: + raise Error("unexpected actor") + return f(*args, **kwargs) -def _get_actor_id(actor: ObjectOrIDType) -> str: - """Helper for retrieving an actor `id`.""" - if isinstance(actor, dict): - return actor['id'] - return actor + return wrapper -class BaseActivity(object): - """Base class for ActivityPub activities.""" +class MicroblogPubBackend(Backend): + def user_agent(self) -> str: + return USER_AGENT - ACTIVITY_TYPE: Optional[ActivityType] = None - ALLOWED_OBJECT_TYPES: List[ActivityType] = [] - OBJECT_REQUIRED = False + def base_url(self) -> str: + return BASE_URL - def __init__(self, **kwargs) -> None: - # Ensure the class has an activity type defined - if not self.ACTIVITY_TYPE: - raise BadActivityError('Missing ACTIVITY_TYPE') + def activity_url(self, obj_id): + return f"{BASE_URL}/outbox/{obj_id}" - # XXX(tsileo): what to do about this check? - # Ensure the activity has a type and a valid one - # if kwargs.get('type') is None: - # raise BadActivityError('missing activity type') - - if kwargs.get('type') and kwargs.pop('type') != self.ACTIVITY_TYPE.value: - raise UnexpectedActivityTypeError(f'Expect the type to be {self.ACTIVITY_TYPE.value!r}') - - # Initialize the object - self._data: Dict[str, Any] = { - 'type': self.ACTIVITY_TYPE.value - } - logger.debug(f'initializing a {self.ACTIVITY_TYPE.value} activity: {kwargs}') - - if 'id' in kwargs: - self._data['id'] = kwargs.pop('id') - - if self.ACTIVITY_TYPE != ActivityType.PERSON: - actor = kwargs.get('actor') - if actor: - kwargs.pop('actor') - actor = self._validate_person(actor) - self._data['actor'] = actor - else: - # FIXME(tsileo): uses a special method to set the actor as "the instance" - if not self.NO_CONTEXT and self.ACTIVITY_TYPE != ActivityType.TOMBSTONE: - actor = ID - self._data['actor'] = actor - - if 'object' in kwargs: - obj = kwargs.pop('object') - if isinstance(obj, str): - self._data['object'] = obj - else: - if not self.ALLOWED_OBJECT_TYPES: - raise UnexpectedActivityTypeError('unexpected object') - if 'type' not in obj or (self.ACTIVITY_TYPE != ActivityType.CREATE and 'id' not in obj): - raise BadActivityError('invalid object, missing type') - if ActivityType(obj['type']) not in self.ALLOWED_OBJECT_TYPES: - raise UnexpectedActivityTypeError( - f'unexpected object type {obj["type"]} (allowed={self.ALLOWED_OBJECT_TYPES})' - ) - self._data['object'] = obj - - if '@context' not in kwargs: - if not self.NO_CONTEXT: - self._data['@context'] = CTX_AS - else: - self._data['@context'] = kwargs.pop('@context') - - # @context check - if not self.NO_CONTEXT: - if not isinstance(self._data['@context'], list): - self._data['@context'] = [self._data['@context']] - if CTX_SECURITY not in self._data['@context']: - self._data['@context'].append(CTX_SECURITY) - if isinstance(self._data['@context'][-1], dict): - self._data['@context'][-1]['Hashtag'] = 'as:Hashtag' - self._data['@context'][-1]['sensitive'] = 'as:sensitive' - else: - self._data['@context'].append({'Hashtag': 'as:Hashtag', 'sensitive': 'as:sensitive'}) - - allowed_keys = None - try: - allowed_keys = self._init(**kwargs) - logger.debug('calling custom init') - except NotImplementedError: - pass - - if allowed_keys: - # Allows an extra to (like for Accept and Follow) - kwargs.pop('to', None) - if len(set(kwargs.keys()) - set(allowed_keys)) > 0: - raise BadActivityError(f'extra data left: {kwargs!r}') - else: - # Remove keys with `None` value - valid_kwargs = {} - for k, v in kwargs.items(): - if v is None: - continue - valid_kwargs[k] = v - self._data.update(**valid_kwargs) - - def _init(self, **kwargs) -> Optional[List[str]]: - raise NotImplementedError - - def _verify(self) -> None: - raise NotImplementedError - - def verify(self) -> None: - """Verifies that the activity is valid.""" - if self.OBJECT_REQUIRED and 'object' not in self._data: - raise BadActivityError('activity must have an "object"') - - try: - self._verify() - except NotImplementedError: - pass - - def __repr__(self) -> str: - return '{}({!r})'.format(self.__class__.__qualname__, self._data.get('id')) - - def __str__(self) -> str: - return str(self._data.get('id', f'[new {self.ACTIVITY_TYPE} activity]')) - - def __getattr__(self, name: str) -> Any: - if self._data.get(name): - return self._data.get(name) - - @property - def type_enum(self) -> ActivityType: - return ActivityType(self.type) - - def _set_id(self, uri: str, obj_id: str) -> None: - raise NotImplementedError - - def set_id(self, uri: str, obj_id: str) -> None: - logger.debug(f'setting ID {uri} / {obj_id}') - self._data['id'] = uri - try: - self._set_id(uri, obj_id) - except NotImplementedError: - pass - - def _actor_id(self, obj: ObjectOrIDType) -> str: - if isinstance(obj, dict) and obj['type'] == ActivityType.PERSON.value: - obj_id = obj.get('id') - if not obj_id: - raise ValueError('missing object id') - return obj_id - else: - return str(obj) - - def _validate_person(self, obj: ObjectOrIDType) -> str: - obj_id = self._actor_id(obj) - try: - actor = ACTOR_SERVICE.get(obj_id) - except Exception: - return obj_id # FIXME(tsileo): handle this - if not actor: - raise ValueError('Invalid actor') - return actor['id'] - - def get_object(self) -> 'BaseActivity': - if self.__obj: - return self.__obj - if isinstance(self._data['object'], dict): - p = parse_activity(self._data['object']) - else: - if self.ACTIVITY_TYPE == ActivityType.FOLLOW: - p = Person(**ACTOR_SERVICE.get(self._data['object'])) - else: - obj = OBJECT_SERVICE.get(self._data['object']) - if ActivityType(obj.get('type')) not in self.ALLOWED_OBJECT_TYPES: - raise UnexpectedActivityTypeError(f'invalid object type {obj.get("type")}') - - p = parse_activity(obj) - - self.__obj: Optional[BaseActivity] = p - return p - - def reset_object_cache(self) -> None: - self.__obj = None - - def to_dict(self, embed: bool = False, embed_object_id_only: bool = False) -> ObjectType: - data = dict(self._data) - if embed: - for k in ['@context', 'signature']: - if k in data: - del(data[k]) - if data.get('object') and embed_object_id_only and isinstance(data['object'], dict): - try: - data['object'] = data['object']['id'] - except KeyError: - raise BadActivityError('embedded object does not have an id') - - return data - - def get_actor(self) -> 'BaseActivity': - actor = self._data.get('actor') - if not actor: - if self.type_enum == ActivityType.NOTE: - actor = str(self._data.get('attributedTo')) - else: - raise ValueError('failed to fetch actor') - - actor_id = self._actor_id(actor) - return Person(**ACTOR_SERVICE.get(actor_id)) - - def _pre_post_to_outbox(self) -> None: - raise NotImplementedError - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - raise NotImplementedError - - def _undo_outbox(self) -> None: - raise NotImplementedError - - def _pre_process_from_inbox(self) -> None: - raise NotImplementedError - - def _process_from_inbox(self) -> None: - raise NotImplementedError - - def _undo_inbox(self) -> None: - raise NotImplementedError - - def _undo_should_purge_cache(self) -> bool: - raise NotImplementedError - - def _should_purge_cache(self) -> bool: - raise NotImplementedError - - def process_from_inbox(self) -> None: - logger.debug(f'calling main process from inbox hook for {self}') - self.verify() - actor = self.get_actor() - - # Check for Block activity - if DB.outbox.find_one({'type': ActivityType.BLOCK.value, - 'activity.object': actor.id, - 'meta.undo': False}): - logger.info(f'actor {actor} is blocked, dropping the received activity {self}') - return - - if DB.inbox.find_one({'remote_id': self.id}): - # The activity is already in the inbox - logger.info(f'received duplicate activity {self}, dropping it') - return - - try: - self._pre_process_from_inbox() - logger.debug('called pre process from inbox hook') - except NotImplementedError: - logger.debug('pre process from inbox hook not implemented') - - activity = self.to_dict() - DB.inbox.insert_one({ - 'activity': activity, - 'type': self.type, - 'remote_id': self.id, - 'meta': {'undo': False, 'deleted': False}, - }) - logger.info('activity {self} saved') - - try: - self._process_from_inbox() - logger.debug('called process from inbox hook') - except NotImplementedError: - logger.debug('process from inbox hook not implemented') - - def post_to_outbox(self) -> None: - logger.debug(f'calling main post to outbox hook for {self}') - obj_id = random_object_id() - self.set_id(f'{ID}/outbox/{obj_id}', obj_id) - self.verify() - - try: - self._pre_post_to_outbox() - logger.debug(f'called pre post to outbox hook') - except NotImplementedError: - logger.debug('pre post to outbox hook not implemented') - - activity = self.to_dict() - DB.outbox.insert_one({ - 'id': obj_id, - 'activity': activity, - 'type': self.type, - 'remote_id': self.id, - 'meta': {'undo': False, 'deleted': False}, - }) - - recipients = self.recipients() - logger.info(f'recipients={recipients}') - activity = clean_activity(activity) - - try: - self._post_to_outbox(obj_id, activity, recipients) - logger.debug(f'called post to outbox hook') - except NotImplementedError: - logger.debug('post to outbox hook not implemented') - - payload = json.dumps(activity) - for recp in recipients: - logger.debug(f'posting to {recp}') - self._post_to_inbox(payload, recp) - - def _post_to_inbox(self, payload: str, to: str): - tasks.post_to_inbox.delay(payload, to) - - def _recipients(self) -> List[str]: - return [] - - def recipients(self) -> List[str]: - recipients = self._recipients() - - out: List[str] = [] - for recipient in recipients: - if recipient in PUBLIC_INSTANCES: - if recipient not in out: - out.append(str(recipient)) - continue - if recipient in [ME, AS_PUBLIC, None]: - continue - if isinstance(recipient, Person): - if recipient.id == ME: - continue - actor = recipient - else: - try: - actor = Person(**ACTOR_SERVICE.get(recipient)) - - if actor.endpoints: - shared_inbox = actor.endpoints.get('sharedInbox') - if shared_inbox not in out: - out.append(shared_inbox) - continue - - if actor.inbox and actor.inbox not in out: - out.append(actor.inbox) - - except NotAnActorError as error: - # Is the activity a `Collection`/`OrderedCollection`? - if error.activity and error.activity['type'] in [ActivityType.COLLECTION.value, - ActivityType.ORDERED_COLLECTION.value]: - for item in parse_collection(error.activity): - if item in [ME, AS_PUBLIC]: - continue - try: - col_actor = Person(**ACTOR_SERVICE.get(item)) - except NotAnActorError: - pass - - if col_actor.endpoints: - shared_inbox = col_actor.endpoints.get('sharedInbox') - if shared_inbox not in out: - out.append(shared_inbox) - continue - if col_actor.inbox and col_actor.inbox not in out: - out.append(col_actor.inbox) - - return out - - def build_undo(self) -> 'BaseActivity': - raise NotImplementedError - - def build_delete(self) -> 'BaseActivity': - raise NotImplementedError - - -class Person(BaseActivity): - ACTIVITY_TYPE = ActivityType.PERSON - - def _init(self, **kwargs): - # if 'icon' in kwargs: - # self._data['icon'] = Image(**kwargs.pop('icon')) - pass - - def _verify(self) -> None: - ACTOR_SERVICE.get(self._data['id']) - - -class Block(BaseActivity): - ACTIVITY_TYPE = ActivityType.BLOCK - OBJECT_REQUIRED = True - - -class Collection(BaseActivity): - ACTIVITY_TYPE = ActivityType.COLLECTION - - -class Image(BaseActivity): - ACTIVITY_TYPE = ActivityType.IMAGE - NO_CONTEXT = True - - def _init(self, **kwargs): - self._data.update( - url=kwargs.pop('url'), + @ensure_it_is_me + def outbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: + DB.outbox.insert_one( + { + "activity": activity.to_dict(), + "type": activity.type, + "remote_id": activity.id, + "meta": {"undo": False, "deleted": False}, + } ) - def __repr__(self): - return 'Image({!r})'.format(self._data.get('url')) - - -class Follow(BaseActivity): - ACTIVITY_TYPE = ActivityType.FOLLOW - ALLOWED_OBJECT_TYPES = [ActivityType.PERSON] - OBJECT_REQUIRED = True - - def _build_reply(self, reply_type: ActivityType) -> BaseActivity: - if reply_type == ActivityType.ACCEPT: - return Accept( - object=self.to_dict(embed=True), + @ensure_it_is_me + def outbox_is_blocked(self, as_actor: ap.Person, actor_id: str) -> bool: + return bool( + DB.outbox.find_one( + { + "type": ap.ActivityType.BLOCK.value, + "activity.object": actor_id, + "meta.undo": False, + } ) + ) - raise ValueError(f'type {reply_type} is invalid for building a reply') + def fetch_iri(self, iri: str) -> ap.ObjectType: + if iri == ME["id"]: + return ME - def _recipients(self) -> List[str]: - return [self.get_object().id] - - def _process_from_inbox(self) -> None: - accept = self.build_accept() - accept.post_to_outbox() - - remote_actor = self.get_actor().id - - if DB.followers.find({'remote_actor': remote_actor}).count() == 0: - DB.followers.insert_one({'remote_actor': remote_actor}) - - def _undo_inbox(self) -> None: - DB.followers.delete_one({'remote_actor': self.get_actor().id}) - - def _undo_outbox(self) -> None: - DB.following.delete_one({'remote_actor': self.get_object().id}) - - def build_accept(self) -> BaseActivity: - return self._build_reply(ActivityType.ACCEPT) - - def build_undo(self) -> BaseActivity: - return Undo(object=self.to_dict(embed=True)) - - def _should_purge_cache(self) -> bool: - # Receiving a follow activity in the inbox should reset the application cache - return True - - -class Accept(BaseActivity): - ACTIVITY_TYPE = ActivityType.ACCEPT - ALLOWED_OBJECT_TYPES = [ActivityType.FOLLOW] - - def _recipients(self) -> List[str]: - return [self.get_object().get_actor().id] - - def _process_from_inbox(self) -> None: - remote_actor = self.get_actor().id - if DB.following.find({'remote_actor': remote_actor}).count() == 0: - DB.following.insert_one({'remote_actor': remote_actor}) - - def _should_purge_cache(self) -> bool: - # Receiving an accept activity in the inbox should reset the application cache - # (a follow request has been accepted) - return True - - -class Undo(BaseActivity): - ACTIVITY_TYPE = ActivityType.UNDO - ALLOWED_OBJECT_TYPES = [ActivityType.FOLLOW, ActivityType.LIKE, ActivityType.ANNOUNCE] - OBJECT_REQUIRED = True - - def _recipients(self) -> List[str]: - obj = self.get_object() - if obj.type_enum == ActivityType.FOLLOW: - return [obj.get_object().id] + # Check if the activity is owned by this server + if iri.startswith(BASE_URL): + is_a_note = False + if iri.endswith("/activity"): + iri = iri.replace("/activity", "") + is_a_note = True + data = DB.outbox.find_one({"remote_id": iri}) + if data: + if is_a_note: + return data["activity"]["object"] + return data["activity"] else: - return [obj.get_object().get_actor().id] - # TODO(tsileo): handle like and announce - raise Exception('TODO') + # Check if the activity is stored in the inbox + data = DB.inbox.find_one({"remote_id": iri}) + if data: + return data["activity"] - def _pre_process_from_inbox(self) -> None: - """Ensures an Undo activity comes from the same actor as the updated activity.""" - obj = self.get_object() - actor = self.get_actor() - if actor.id != obj.get_actor().id: - raise BadActivityError(f'{actor!r} cannot update {obj!r}') + # Fetch the URL via HTTP + return super().fetch_iri(iri) - def _process_from_inbox(self) -> None: - obj = self.get_object() - DB.inbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) + @ensure_it_is_me + def inbox_check_duplicate(self, as_actor: ap.Person, iri: str) -> bool: + return bool(DB.inbox.find_one({"remote_id": iri})) - try: - obj._undo_inbox() - except NotImplementedError: - pass + @ensure_it_is_me + def inbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: + DB.inbox.insert_one( + { + "activity": activity.to_dict(), + "type": activity.type, + "remote_id": activity.id, + "meta": {"undo": False, "deleted": False}, + } + ) - def _should_purge_cache(self) -> bool: - obj = self.get_object() - try: - # Receiving a undo activity regarding an activity that was mentioning a published activity - # should purge the cache - return obj._undo_should_purge_cache() - except NotImplementedError: - pass + @ensure_it_is_me + def post_to_remote_inbox(self, as_actor: ap.Person, payload: str, to: str) -> None: + tasks.post_to_inbox.delay(payload, to) - return False + @ensure_it_is_me + def new_follower(self, as_actor: ap.Person, follow: ap.Follow) -> None: + remote_actor = follow.get_actor().id - def _pre_post_to_outbox(self) -> None: - """Ensures an Undo activity references an activity owned by the instance.""" - obj = self.get_object() - if not obj.id.startswith(ID): - raise NotFromOutboxError(f'object {obj["id"]} is not owned by this instance') + if DB.followers.find({"remote_actor": remote_actor}).count() == 0: + DB.followers.insert_one({"remote_actor": remote_actor}) - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - logger.debug('processing undo to outbox') - logger.debug('self={}'.format(self)) - obj = self.get_object() - logger.debug('obj={}'.format(obj)) - DB.outbox.update_one({'remote_id': obj.id}, {'$set': {'meta.undo': True}}) + @ensure_it_is_me + def undo_new_follower(self, as_actor: ap.Person, follow: ap.Follow) -> None: + # TODO(tsileo): update the follow to set undo + DB.followers.delete_one({"remote_actor": follow.get_actor().id}) - try: - obj._undo_outbox() - logger.debug(f'_undo_outbox called for {obj}') - except NotImplementedError: - logger.debug(f'_undo_outbox not implemented for {obj}') - pass + @ensure_it_is_me + def undo_new_following(self, as_actor: ap.Person, follow: ap.Follow) -> None: + # TODO(tsileo): update the follow to set undo + DB.following.delete_one({"remote_actor": follow.get_object().id}) + @ensure_it_is_me + def new_following(self, as_actor: ap.Person, follow: ap.Follow) -> None: + remote_actor = follow.get_object().id + if DB.following.find({"remote_actor": remote_actor}).count() == 0: + DB.following.insert_one({"remote_actor": remote_actor}) -class Like(BaseActivity): - ACTIVITY_TYPE = ActivityType.LIKE - ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] - OBJECT_REQUIRED = True - - def _recipients(self) -> List[str]: - return [self.get_object().get_actor().id] - - def _process_from_inbox(self): - obj = self.get_object() + @ensure_it_is_me + def inbox_like(self, as_actor: ap.Person, like: ap.Like) -> None: + obj = like.get_object() # Update the meta counter if the object is published by the server - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_like': 1}, - }) - # XXX(tsileo): notification?? + DB.outbox.update_one( + {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": 1}} + ) - def _undo_inbox(self) -> None: - obj = self.get_object() + @ensure_it_is_me + def inbox_undo_like(self, as_actor: ap.Person, like: ap.Like) -> None: + obj = like.get_object() # Update the meta counter if the object is published by the server - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_like': -1}, - }) + DB.outbox.update_one( + {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": -1}} + ) - def _undo_should_purge_cache(self) -> bool: - # If a like coutn was decremented, we need to purge the application cache - return self.get_object().id.startswith(BASE_URL) - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]): - obj = self.get_object() + @ensure_it_is_me + def outbox_like(self, as_actor: ap.Person, like: ap.Like) -> None: + obj = like.get_object() # Unlikely, but an actor can like it's own post - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_like': 1}, - }) + DB.outbox.update_one( + {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": 1}} + ) # Keep track of the like we just performed - DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.liked': obj_id}}) + DB.inbox.update_one( + {"activity.object.id": obj.id}, {"$set": {"meta.liked": like.id}} + ) - def _undo_outbox(self) -> None: - obj = self.get_object() + @ensure_it_is_me + def outbox_undo_like(self, as_actor: ap.Person, like: ap.Like) -> None: + obj = like.get_object() # Unlikely, but an actor can like it's own post - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_like': -1}, - }) + DB.outbox.update_one( + {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": -1}} + ) - DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.liked': False}}) + DB.inbox.update_one( + {"activity.object.id": obj.id}, {"$set": {"meta.liked": False}} + ) - def build_undo(self) -> BaseActivity: - return Undo(object=self.to_dict(embed=True, embed_object_id_only=True)) - - -class Announce(BaseActivity): - ACTIVITY_TYPE = ActivityType.ANNOUNCE - ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] - - def _recipients(self) -> List[str]: - recipients = [] - - for field in ['to', 'cc']: - if field in self._data: - recipients.extend(_to_list(self._data[field])) - - return recipients - - def _process_from_inbox(self) -> None: - if isinstance(self._data['object'], str) and not self._data['object'].startswith('http'): + @ensure_it_is_me + def inbox_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: + if isinstance(announce._data["object"], str) and not announce._data[ + "object" + ].startswith("http"): # TODO(tsileo): actually drop it without storing it and better logging, also move the check somewhere else logger.warn( - f'received an Annouce referencing an OStatus notice ({self._data["object"]}), dropping the message' + f'received an Annouce referencing an OStatus notice ({announce._data["object"]}), dropping the message' ) return - # Save/cache the object, and make it part of the stream so we can fetch it - if isinstance(self._data['object'], str): - raw_obj = OBJECT_SERVICE.get( - self._data['object'], - reload_cache=True, - part_of_stream=True, - announce_published=self._data['published'], - ) - obj = parse_activity(raw_obj) + # FIXME(tsileo): Save/cache the object, and make it part of the stream so we can fetch it + if isinstance(announce._data["object"], str): + obj_iri = announce._data["object"] else: - obj = self.get_object() + obj_iri = self.get_object().id - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_boost': 1}, - }) + DB.outbox.update_one( + {"activity.object.id": obj_iri}, {"$inc": {"meta.count_boost": 1}} + ) - def _undo_inbox(self) -> None: - obj = self.get_object() + @ensure_it_is_me + def inbox_undo_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: + obj = announce.get_object() # Update the meta counter if the object is published by the server - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$inc': {'meta.count_boost': -1}, - }) + DB.outbox.update_one( + {"activity.object.id": obj.id}, {"$inc": {"meta.count_boost": -1}} + ) - def _undo_should_purge_cache(self) -> bool: - # If a like coutn was decremented, we need to purge the application cache - return self.get_object().id.startswith(BASE_URL) + @ensure_it_is_me + def outbox_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: + obj = announce.get_object() + DB.inbox.update_one( + {"activity.object.id": obj.id}, {"$set": {"meta.boosted": announce.id}} + ) - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - if isinstance(self._data['object'], str): - # Put the object in the cache - OBJECT_SERVICE.get( - self._data['object'], - reload_cache=True, - part_of_stream=True, - announce_published=self._data['published'], - ) + @ensure_it_is_me + def outbox_undo_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: + obj = announce.get_object() + DB.inbox.update_one( + {"activity.object.id": obj.id}, {"$set": {"meta.boosted": False}} + ) - obj = self.get_object() - DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.boosted': obj_id}}) + @ensure_it_is_me + def inbox_delete(self, as_actor: ap.Person, delete: ap.Delete) -> None: + DB.inbox.update_one( + {"activity.object.id": delete.get_object().id}, + {"$set": {"meta.deleted": True}}, + ) + obj = delete.get_object() + if obj.ACTIVITY_TYPE != ap.ActivityType.NOTE: + obj = ap.parse_activity( + DB.inbox.find_one( + { + "activity.object.id": delete.get_object().id, + "type": ap.ActivityType.CREATE.value, + } + )["activity"] + ).get_object() - def _undo_outbox(self) -> None: - obj = self.get_object() - DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'meta.boosted': False}}) + logger.info(f"inbox_delete handle_replies obj={obj!r}") + if obj: + self._handle_replies_delete(as_actor, obj) - def build_undo(self) -> BaseActivity: - return Undo(object=self.to_dict(embed=True)) - - -class Delete(BaseActivity): - ACTIVITY_TYPE = ActivityType.DELETE - ALLOWED_OBJECT_TYPES = [ActivityType.NOTE, ActivityType.TOMBSTONE] - OBJECT_REQUIRED = True - - def _get_actual_object(self) -> BaseActivity: - obj = self.get_object() - if obj.type_enum == ActivityType.TOMBSTONE: - obj = parse_activity(OBJECT_SERVICE.get(obj.id)) - return obj - - def _recipients(self) -> List[str]: - obj = self._get_actual_object() - return obj._recipients() - - def _pre_process_from_inbox(self) -> None: - """Ensures a Delete activity comes from the same actor as the deleted activity.""" - obj = self._get_actual_object() - actor = self.get_actor() - if actor.id != obj.get_actor().id: - raise BadActivityError(f'{actor!r} cannot delete {obj!r}') - - def _process_from_inbox(self) -> None: - DB.inbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) - obj = self._get_actual_object() - if obj.type_enum == ActivityType.NOTE: - obj._delete_from_threads() + # FIXME(tsileo): handle threads + # obj = delete._get_actual_object() + # if obj.type_enum == ActivityType.NOTE: + # obj._delete_from_threads() # TODO(tsileo): also purge the cache if it's a reply of a published activity - def _pre_post_to_outbox(self) -> None: - """Ensures the Delete activity references a activity from the outbox (i.e. owned by the instance).""" - obj = self._get_actual_object() - if not obj.id.startswith(ID): - raise NotFromOutboxError(f'object {obj["id"]} is not owned by this instance') + @ensure_it_is_me + def outbox_delete(self, as_actor: ap.Person, delete: ap.Delete) -> None: + DB.outbox.update_one( + {"activity.object.id": delete.get_object().id}, + {"$set": {"meta.deleted": True}}, + ) + obj = delete.get_object() + if delete.get_object().ACTIVITY_TYPE != ap.ActivityType.NOTE: + obj = ap.parse_activity( + DB.outbox.find_one( + { + "activity.object.id": delete.get_object().id, + "type": ap.ActivityType.CREATE.value, + } + )["activity"] + ).get_object() - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - DB.outbox.update_one({'activity.object.id': self.get_object().id}, {'$set': {'meta.deleted': True}}) + self._handle_replies_delete(as_actor, obj) - -class Update(BaseActivity): - ACTIVITY_TYPE = ActivityType.UPDATE - ALLOWED_OBJECT_TYPES = [ActivityType.NOTE, ActivityType.PERSON] - OBJECT_REQUIRED = True - - def _pre_process_from_inbox(self) -> None: - """Ensures an Update activity comes from the same actor as the updated activity.""" - obj = self.get_object() - actor = self.get_actor() - if actor.id != obj.get_actor().id: - raise BadActivityError(f'{actor!r} cannot update {obj!r}') - - def _process_from_inbox(self): - obj = self.get_object() - if obj.type_enum == ActivityType.NOTE: - DB.inbox.update_one({'activity.object.id': obj.id}, {'$set': {'activity.object': obj.to_dict()}}) + @ensure_it_is_me + def inbox_update(self, as_actor: ap.Person, update: ap.Update) -> None: + obj = update.get_object() + if obj.ACTIVITY_TYPE == ap.ActivityType.NOTE: + DB.inbox.update_one( + {"activity.object.id": obj.id}, + {"$set": {"activity.object": obj.to_dict()}}, + ) return - # If the object is a Person, it means the profile was updated, we just refresh our local cache - ACTOR_SERVICE.get(obj.id, reload_cache=True) + # FIXME(tsileo): handle update actor amd inbox_update_note/inbox_update_actor - # TODO(tsileo): implements _should_purge_cache if it's a reply of a published activity (i.e. in the outbox) + @ensure_it_is_me + def outbox_update(self, as_actor: ap.Person, _update: ap.Update) -> None: + obj = _update._data["object"] - def _pre_post_to_outbox(self) -> None: - obj = self.get_object() - if not obj.id.startswith(ID): - raise NotFromOutboxError(f'object {obj["id"]} is not owned by this instance') - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - obj = self._data['object'] - - update_prefix = 'activity.object.' - update: Dict[str, Any] = {'$set': dict(), '$unset': dict()} - update['$set'][f'{update_prefix}updated'] = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' + update_prefix = "activity.object." + update: Dict[str, Any] = {"$set": dict(), "$unset": dict()} + update["$set"][f"{update_prefix}updated"] = ( + datetime.utcnow().replace(microsecond=0).isoformat() + "Z" + ) for k, v in obj.items(): - if k in ['id', 'type']: + if k in ["id", "type"]: continue if v is None: - update['$unset'][f'{update_prefix}{k}'] = '' + update["$unset"][f"{update_prefix}{k}"] = "" else: - update['$set'][f'{update_prefix}{k}'] = v + update["$set"][f"{update_prefix}{k}"] = v - if len(update['$unset']) == 0: - del(update['$unset']) + if len(update["$unset"]) == 0: + del (update["$unset"]) - print(f'updating note from outbox {obj!r} {update}') - logger.info(f'updating note from outbox {obj!r} {update}') - DB.outbox.update_one({'activity.object.id': obj['id']}, update) + print(f"updating note from outbox {obj!r} {update}") + logger.info(f"updating note from outbox {obj!r} {update}") + DB.outbox.update_one({"activity.object.id": obj["id"]}, update) # FIXME(tsileo): should send an Update (but not a partial one, to all the note's recipients # (create a new Update with the result of the update, and send it without saving it?) + @ensure_it_is_me + def outbox_create(self, as_actor: ap.Person, create: ap.Create) -> None: + self._handle_replies(as_actor, create) -class Create(BaseActivity): - ACTIVITY_TYPE = ActivityType.CREATE - ALLOWED_OBJECT_TYPES = [ActivityType.NOTE] - OBJECT_REQUIRED = True + @ensure_it_is_me + def inbox_create(self, as_actor: ap.Person, create: ap.Create) -> None: + self._handle_replies(as_actor, create) - def _set_id(self, uri: str, obj_id: str) -> None: - self._data['object']['id'] = uri + '/activity' - self._data['object']['url'] = ID + '/' + self.get_object().type.lower() + '/' + obj_id - self.reset_object_cache() - - def _init(self, **kwargs): - obj = self.get_object() - if not obj.attributedTo: - self._data['object']['attributedTo'] = self.get_actor().id - if not obj.published: - if self.published: - self._data['object']['published'] = self.published - else: - now = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' - self._data['published'] = now - self._data['object']['published'] = now - - def _recipients(self) -> List[str]: - # TODO(tsileo): audience support? - recipients = [] - for field in ['to', 'cc', 'bto', 'bcc']: - if field in self._data: - recipients.extend(_to_list(self._data[field])) - - recipients.extend(self.get_object()._recipients()) - - return recipients - - def _update_threads(self) -> None: - logger.debug('_update_threads hook') - obj = self.get_object() - - # TODO(tsileo): re-enable me - # tasks.fetch_og.delay('INBOX', self.id) - - threads = [] - reply = obj.get_local_reply() - print(f'initial_reply={reply}') - print(f'{obj}') - logger.debug(f'initial_reply={reply}') - reply_id = None - direct_reply = 1 - while reply is not None: - if not DB.inbox.find_one_and_update({'activity.object.id': reply.id}, { - '$inc': { - 'meta.count_reply': 1, - 'meta.count_direct_reply': direct_reply, - }, - '$addToSet': {'meta.thread_children': obj.id}, - }): - DB.outbox.update_one({'activity.object.id': reply.id}, { - '$inc': { - 'meta.count_reply': 1, - 'meta.count_direct_reply': direct_reply, - }, - '$addToSet': {'meta.thread_children': obj.id}, - }) - - direct_reply = 0 - reply_id = reply.id - reply = reply.get_local_reply() - logger.debug(f'next_reply={reply}') - threads.append(reply_id) - # FIXME(tsileo): obj.id is None!! - print(f'reply_id={reply_id} {obj.id} {obj._data} {self.id}') - - if reply_id: - if not DB.inbox.find_one_and_update({'activity.object.id': obj.id}, { - '$set': { - 'meta.thread_parents': threads, - 'meta.thread_root_parent': reply_id, - }, - }): - DB.outbox.update_one({'activity.object.id': obj.id}, { - '$set': { - 'meta.thread_parents': threads, - 'meta.thread_root_parent': reply_id, - }, - }) - logger.debug('_update_threads done') - - def _process_from_inbox(self) -> None: - self._update_threads() - - def _post_to_outbox(self, obj_id: str, activity: ObjectType, recipients: List[str]) -> None: - self._update_threads() - - def _should_purge_cache(self) -> bool: - # TODO(tsileo): handle reply of a reply... - obj = self.get_object() - in_reply_to = obj.inReplyTo - if in_reply_to: - local_activity = DB.outbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) - if local_activity: - return True - - return False - - -class Tombstone(BaseActivity): - ACTIVITY_TYPE = ActivityType.TOMBSTONE - - -class Note(BaseActivity): - ACTIVITY_TYPE = ActivityType.NOTE - - def _init(self, **kwargs): - print(self._data) - # Remove the `actor` field as `attributedTo` is used for `Note` instead - if 'actor' in self._data: - del(self._data['actor']) - if 'sensitive' not in kwargs: - self._data['sensitive'] = False - - def _recipients(self) -> List[str]: - # TODO(tsileo): audience support? - recipients: List[str] = [] - - # If the note is public, we publish it to the defined "public instances" - if AS_PUBLIC in self._data.get('to', []): - recipients.extend(PUBLIC_INSTANCES) - print('publishing to public instances') - print(recipients) - - for field in ['to', 'cc', 'bto', 'bcc']: - if field in self._data: - recipients.extend(_to_list(self._data[field])) - - return recipients - - def _delete_from_threads(self) -> None: - logger.debug('_delete_from_threads hook') - - reply = self.get_local_reply() - logger.debug(f'initial_reply={reply}') - direct_reply = -1 - while reply is not None: - if not DB.inbox.find_one_and_update({'activity.object.id': reply.id}, { - '$inc': { - 'meta.count_reply': -1, - 'meta.count_direct_reply': direct_reply, - }, - '$pull': {'meta.thread_children': self.id}, - - }): - DB.outbox.update_one({'activity.object.id': reply.id}, { - '$inc': { - 'meta.count_reply': 1, - 'meta.count_direct_reply': direct_reply, - }, - '$pull': {'meta.thread_children': self.id}, - }) - - direct_reply = 0 - reply = reply.get_local_reply() - logger.debug(f'next_reply={reply}') - - logger.debug('_delete_from_threads done') - return None - - def get_local_reply(self) -> Optional[BaseActivity]: - "Find the note reply if any.""" - in_reply_to = self.inReplyTo + @ensure_it_is_me + def _handle_replies_delete(self, as_actor: ap.Person, note: ap.Note) -> None: + in_reply_to = note.inReplyTo if not in_reply_to: - # This is the root comment - return None + pass - inbox_parent = DB.inbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) - if inbox_parent: - return parse_activity(inbox_parent['activity']['object']) + if not DB.inbox.find_one_and_update( + {"activity.object.id": in_reply_to}, + {"$inc": {"meta.count_reply": -1, "meta.count_direct_reply": -1}}, + ): + DB.outbox.update_one( + {"activity.object.id": in_reply_to}, + {"$inc": {"meta.count_reply": -1, "meta.count_direct_reply": -1}}, + ) - outbox_parent = DB.outbox.find_one({'activity.type': 'Create', 'activity.object.id': in_reply_to}) - if outbox_parent: - return parse_activity(outbox_parent['activity']['object']) + @ensure_it_is_me + def _handle_replies(self, as_actor: ap.Person, create: ap.Create) -> None: + in_reply_to = create.get_object().inReplyTo + if not in_reply_to: + pass - # The parent is no stored on this instance - return None - - def build_create(self) -> BaseActivity: - """Wraps an activity in a Create activity.""" - create_payload = { - 'object': self.to_dict(embed=True), - 'actor': self.attributedTo or ME, - } - for field in ['published', 'to', 'bto', 'cc', 'bcc', 'audience']: - if field in self._data: - create_payload[field] = self._data[field] - - return Create(**create_payload) - - def build_like(self) -> BaseActivity: - return Like(object=self.id) - - def build_announce(self) -> BaseActivity: - return Announce( - object=self.id, - to=[AS_PUBLIC], - cc=[ID+'/followers', self.attributedTo], - published=datetime.utcnow().replace(microsecond=0).isoformat() + 'Z', - ) - - def build_delete(self) -> BaseActivity: - return Delete(object=Tombstone(id=self.id).to_dict(embed=True)) - - def get_tombstone(self, deleted: Optional[str]) -> BaseActivity: - return Tombstone( - id=self.id, - published=self.published, - deleted=deleted, - updated=deleted, - ) - - -_ACTIVITY_TYPE_TO_CLS = { - ActivityType.IMAGE: Image, - ActivityType.PERSON: Person, - ActivityType.FOLLOW: Follow, - ActivityType.ACCEPT: Accept, - ActivityType.UNDO: Undo, - ActivityType.LIKE: Like, - ActivityType.ANNOUNCE: Announce, - ActivityType.UPDATE: Update, - ActivityType.DELETE: Delete, - ActivityType.CREATE: Create, - ActivityType.NOTE: Note, - ActivityType.BLOCK: Block, - ActivityType.COLLECTION: Collection, - ActivityType.TOMBSTONE: Tombstone, -} - - -def parse_activity(payload: ObjectType, expected: Optional[ActivityType] = None) -> BaseActivity: - t = ActivityType(payload['type']) - - if expected and t != expected: - raise UnexpectedActivityTypeError(f'expected a {expected.name} activity, got a {payload["type"]}') - - if t not in _ACTIVITY_TYPE_TO_CLS: - raise BadActivityError(f'unsupported activity type {payload["type"]}') - - activity = _ACTIVITY_TYPE_TO_CLS[t](**payload) - - return activity + if not DB.inbox.find_one_and_update( + {"activity.object.id": in_reply_to}, + {"$inc": {"meta.count_reply": 1, "meta.count_direct_reply": 1}}, + ): + DB.outbox.update_one( + {"activity.object.id": in_reply_to}, + {"$inc": {"meta.count_reply": 1, "meta.count_direct_reply": 1}}, + ) def gen_feed(): fg = FeedGenerator() - fg.id(f'{ID}') - fg.title(f'{USERNAME} notes') - fg.author({'name': USERNAME, 'email': 't@a4.io'}) - fg.link(href=ID, rel='alternate') - fg.description(f'{USERNAME} notes') - fg.logo(ME.get('icon', {}).get('url')) - fg.language('en') - for item in DB.outbox.find({'type': 'Create'}, limit=50): + fg.id(f"{ID}") + fg.title(f"{USERNAME} notes") + fg.author({"name": USERNAME, "email": "t@a4.io"}) + fg.link(href=ID, rel="alternate") + fg.description(f"{USERNAME} notes") + fg.logo(ME.get("icon", {}).get("url")) + fg.language("en") + for item in DB.outbox.find({"type": "Create"}, limit=50): fe = fg.add_entry() - fe.id(item['activity']['object'].get('url')) - fe.link(href=item['activity']['object'].get('url')) - fe.title(item['activity']['object']['content']) - fe.description(item['activity']['object']['content']) + fe.id(item["activity"]["object"].get("url")) + fe.link(href=item["activity"]["object"].get("url")) + fe.title(item["activity"]["object"]["content"]) + fe.description(item["activity"]["object"]["content"]) return fg def json_feed(path: str) -> Dict[str, Any]: """JSON Feed (https://jsonfeed.org/) document.""" data = [] - for item in DB.outbox.find({'type': 'Create'}, limit=50): - data.append({ - "id": item["id"], - "url": item['activity']['object'].get('url'), - "content_html": item['activity']['object']['content'], - "content_text": html2text(item['activity']['object']['content']), - "date_published": item['activity']['object'].get('published'), - }) + for item in DB.outbox.find({"type": "Create"}, limit=50): + data.append( + { + "id": item["id"], + "url": item["activity"]["object"].get("url"), + "content_html": item["activity"]["object"]["content"], + "content_text": html2text(item["activity"]["object"]["content"]), + "date_published": item["activity"]["object"].get("published"), + } + ) return { "version": "https://jsonfeed.org/version/1", - "user_comment": ("This is a microblog feed. You can add this to your feed reader using the following URL: " - + ID + path), + "user_comment": ( + "This is a microblog feed. You can add this to your feed reader using the following URL: " + + ID + + path + ), "title": USERNAME, "home_page_url": ID, "feed_url": ID + path, "author": { "name": USERNAME, "url": ID, - "avatar": ME.get('icon', {}).get('url'), + "avatar": ME.get("icon", {}).get("url"), }, "items": data, } -def build_inbox_json_feed(path: str, request_cursor: Optional[str] = None) -> Dict[str, Any]: +def build_inbox_json_feed( + path: str, request_cursor: Optional[str] = None +) -> Dict[str, Any]: data = [] cursor = None - q: Dict[str, Any] = {'type': 'Create', 'meta.deleted': False} + q: Dict[str, Any] = {"type": "Create", "meta.deleted": False} if request_cursor: - q['_id'] = {'$lt': request_cursor} + q["_id"] = {"$lt": request_cursor} - for item in DB.inbox.find(q, limit=50).sort('_id', -1): - actor = ACTOR_SERVICE.get(item['activity']['actor']) - data.append({ - "id": item["activity"]["id"], - "url": item['activity']['object'].get('url'), - "content_html": item['activity']['object']['content'], - "content_text": html2text(item['activity']['object']['content']), - "date_published": item['activity']['object'].get('published'), - "author": { - "name": actor.get('name', actor.get('preferredUsername')), - "url": actor.get('url'), - 'avatar': actor.get('icon', {}).get('url'), - }, - }) - cursor = str(item['_id']) + for item in DB.inbox.find(q, limit=50).sort("_id", -1): + actor = ap.get_backend().fetch_iri(item["activity"]["actor"]) + data.append( + { + "id": item["activity"]["id"], + "url": item["activity"]["object"].get("url"), + "content_html": item["activity"]["object"]["content"], + "content_text": html2text(item["activity"]["object"]["content"]), + "date_published": item["activity"]["object"].get("published"), + "author": { + "name": actor.get("name", actor.get("preferredUsername")), + "url": actor.get("url"), + "avatar": actor.get("icon", {}).get("url"), + }, + } + ) + cursor = str(item["_id"]) resp = { "version": "https://jsonfeed.org/version/1", - "title": f'{USERNAME}\'s stream', + "title": f"{USERNAME}'s stream", "home_page_url": ID, "feed_url": ID + path, "items": data, } if cursor and len(data) == 50: - resp['next_url'] = ID + path + '?cursor=' + cursor + resp["next_url"] = ID + path + "?cursor=" + cursor return resp -def parse_collection(payload: Optional[Dict[str, Any]] = None, url: Optional[str] = None) -> List[str]: +def parse_collection( + payload: Optional[Dict[str, Any]] = None, url: Optional[str] = None +) -> List[str]: """Resolve/fetch a `Collection`/`OrderedCollection`.""" # Resolve internal collections via MongoDB directly - if url == ID + '/followers': - return [doc['remote_actor'] for doc in DB.followers.find()] - elif url == ID + '/following': - return [doc['remote_actor'] for doc in DB.following.find()] + if url == ID + "/followers": + return [doc["remote_actor"] for doc in DB.followers.find()] + elif url == ID + "/following": + return [doc["remote_actor"] for doc in DB.following.find()] # Go through all the pages - return activitypub_utils.parse_collection(payload, url) + return ap_parse_collection(payload, url) def embed_collection(total_items, first_page_id): return { - "type": ActivityType.ORDERED_COLLECTION.value, + "type": ap.ActivityType.ORDERED_COLLECTION.value, "totalItems": total_items, - "first": f'{first_page_id}?page=first', + "first": f"{first_page_id}?page=first", "id": first_page_id, } -def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, col_name=None, first_page=False): +def build_ordered_collection( + col, q=None, cursor=None, map_func=None, limit=50, col_name=None, first_page=False +): col_name = col_name or col.name if q is None: q = {} if cursor: - q['_id'] = {'$lt': ObjectId(cursor)} - data = list(col.find(q, limit=limit).sort('_id', -1)) + q["_id"] = {"$lt": ObjectId(cursor)} + data = list(col.find(q, limit=limit).sort("_id", -1)) if not data: return { - 'id': BASE_URL + '/' + col_name, - 'totalItems': 0, - 'type': ActivityType.ORDERED_COLLECTION.value, - 'orederedItems': [], + "id": BASE_URL + "/" + col_name, + "totalItems": 0, + "type": ap.ActivityType.ORDERED_COLLECTION.value, + "orederedItems": [], } - start_cursor = str(data[0]['_id']) - next_page_cursor = str(data[-1]['_id']) + start_cursor = str(data[0]["_id"]) + next_page_cursor = str(data[-1]["_id"]) total_items = col.find(q).count() data = [_remove_id(doc) for doc in data] @@ -1210,41 +501,43 @@ def build_ordered_collection(col, q=None, cursor=None, map_func=None, limit=50, # No cursor, this is the first page and we return an OrderedCollection if not cursor: resp = { - '@context': COLLECTION_CTX, - 'id': f'{BASE_URL}/{col_name}', - 'totalItems': total_items, - 'type': ActivityType.ORDERED_COLLECTION.value, - 'first': { - 'id': f'{BASE_URL}/{col_name}?cursor={start_cursor}', - 'orderedItems': data, - 'partOf': f'{BASE_URL}/{col_name}', - 'totalItems': total_items, - 'type': ActivityType.ORDERED_COLLECTION_PAGE.value, + "@context": ap.COLLECTION_CTX, + "id": f"{BASE_URL}/{col_name}", + "totalItems": total_items, + "type": ap.ActivityType.ORDERED_COLLECTION.value, + "first": { + "id": f"{BASE_URL}/{col_name}?cursor={start_cursor}", + "orderedItems": data, + "partOf": f"{BASE_URL}/{col_name}", + "totalItems": total_items, + "type": ap.ActivityType.ORDERED_COLLECTION_PAGE.value, }, } if len(data) == limit: - resp['first']['next'] = BASE_URL + '/' + col_name + '?cursor=' + next_page_cursor + resp["first"]["next"] = ( + BASE_URL + "/" + col_name + "?cursor=" + next_page_cursor + ) if first_page: - return resp['first'] + return resp["first"] return resp # If there's a cursor, then we return an OrderedCollectionPage resp = { - '@context': COLLECTION_CTX, - 'type': ActivityType.ORDERED_COLLECTION_PAGE.value, - 'id': BASE_URL + '/' + col_name + '?cursor=' + start_cursor, - 'totalItems': total_items, - 'partOf': BASE_URL + '/' + col_name, - 'orderedItems': data, + "@context": ap.COLLECTION_CTX, + "type": ap.ActivityType.ORDERED_COLLECTION_PAGE.value, + "id": BASE_URL + "/" + col_name + "?cursor=" + start_cursor, + "totalItems": total_items, + "partOf": BASE_URL + "/" + col_name, + "orderedItems": data, } if len(data) == limit: - resp['next'] = BASE_URL + '/' + col_name + '?cursor=' + next_page_cursor + resp["next"] = BASE_URL + "/" + col_name + "?cursor=" + next_page_cursor if first_page: - return resp['first'] + return resp["first"] # XXX(tsileo): implements prev with prev=? diff --git a/app.py b/app.py index e3cd805..fd564c1 100644 --- a/app.py +++ b/app.py @@ -1,135 +1,145 @@ import binascii import hashlib import json -import urllib -import os -import mimetypes import logging -from functools import wraps +import mimetypes +import os +import urllib from datetime import datetime +from functools import wraps +from typing import Any +from typing import Dict +from urllib.parse import urlencode +from urllib.parse import urlparse -import timeago import bleach import mf2py -import pymongo import piexif +import pymongo +import timeago from bson.objectid import ObjectId from flask import Flask -from flask import abort -from flask import request -from flask import redirect from flask import Response -from flask import render_template -from flask import session +from flask import abort from flask import jsonify as flask_jsonify +from flask import redirect +from flask import render_template +from flask import request +from flask import session from flask import url_for +from flask_wtf.csrf import CSRFProtect from html2text import html2text -from itsdangerous import JSONWebSignatureSerializer from itsdangerous import BadSignature from passlib.hash import bcrypt from u2flib_server import u2f -from urllib.parse import urlparse, urlencode from werkzeug.utils import secure_filename -from flask_wtf.csrf import CSRFProtect import activitypub import config -from activitypub import ActivityType -from activitypub import clean_activity from activitypub import embed_collection -from utils.content_helper import parse_markdown -from config import KEY -from config import DB -from config import ME -from config import ID -from config import DOMAIN -from config import USERNAME -from config import BASE_URL -from config import ACTOR_SERVICE -from config import OBJECT_SERVICE -from config import PASS -from config import HEADERS -from config import VERSION -from config import DEBUG_MODE -from config import JWT from config import ADMIN_API_KEY +from config import BASE_URL +from config import DB +from config import DEBUG_MODE +from config import DOMAIN +from config import HEADERS +from config import ID +from config import JWT +from config import KEY +from config import ME +from config import PASS +from config import USERNAME +from config import VERSION from config import _drop_db from config import custom_cache_purge_hook -from utils.httpsig import HTTPSigAuth, verify_request +from little_boxes import activitypub as ap +from little_boxes.activitypub import ActivityType +from little_boxes.activitypub import clean_activity +from little_boxes.content_helper import parse_markdown +from little_boxes.errors import ActivityNotFoundError +from little_boxes.errors import Error +from little_boxes.errors import NotFromOutboxError +from little_boxes.httpsig import HTTPSigAuth + +# from little_boxes.httpsig import verify_request +from little_boxes.webfinger import get_actor_url +from little_boxes.webfinger import get_remote_follow_template from utils.key import get_secret_key -from utils.webfinger import get_remote_follow_template -from utils.webfinger import get_actor_url -from utils.errors import Error -from utils.errors import UnexpectedActivityTypeError -from utils.errors import BadActivityError -from utils.errors import NotFromOutboxError -from utils.errors import ActivityNotFoundError +from utils.object_service import ObjectService +OBJECT_SERVICE = ACTOR_SERVICE = ObjectService() + +back = activitypub.MicroblogPubBackend() +ap.use_backend(back) + +MY_PERSON = ap.Person(**ME) -from typing import Dict, Any - app = Flask(__name__) -app.secret_key = get_secret_key('flask') -app.config.update( - WTF_CSRF_CHECK_DEFAULT=False, -) +app.secret_key = get_secret_key("flask") +app.config.update(WTF_CSRF_CHECK_DEFAULT=False) csrf = CSRFProtect(app) logger = logging.getLogger(__name__) # Hook up Flask logging with gunicorn root_logger = logging.getLogger() -if os.getenv('FLASK_DEBUG'): +if os.getenv("FLASK_DEBUG"): logger.setLevel(logging.DEBUG) root_logger.setLevel(logging.DEBUG) else: - gunicorn_logger = logging.getLogger('gunicorn.error') + gunicorn_logger = logging.getLogger("gunicorn.error") root_logger.handlers = gunicorn_logger.handlers root_logger.setLevel(gunicorn_logger.level) -SIG_AUTH = HTTPSigAuth(ID+'#main-key', KEY.privkey) +SIG_AUTH = HTTPSigAuth(KEY) + +OUTBOX = ap.Outbox(MY_PERSON) +INBOX = ap.Inbox(MY_PERSON) def verify_pass(pwd): - return bcrypt.verify(pwd, PASS) + return bcrypt.verify(pwd, PASS) + @app.context_processor def inject_config(): - return dict( - microblogpub_version=VERSION, - config=config, - logged_in=session.get('logged_in', False), - ) + return dict( + microblogpub_version=VERSION, + config=config, + logged_in=session.get("logged_in", False), + ) + @app.after_request def set_x_powered_by(response): - response.headers['X-Powered-By'] = 'microblog.pub' + response.headers["X-Powered-By"] = "microblog.pub" return response + # HTML/templates helper ALLOWED_TAGS = [ - 'a', - 'abbr', - 'acronym', - 'b', - 'blockquote', - 'code', - 'pre', - 'em', - 'i', - 'li', - 'ol', - 'strong', - 'ul', - 'span', - 'div', - 'p', - 'h1', - 'h2', - 'h3', - 'h4', - 'h5', - 'h6', + "a", + "abbr", + "acronym", + "b", + "blockquote", + "code", + "pre", + "em", + "i", + "li", + "ol", + "strong", + "ul", + "span", + "div", + "p", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", ] @@ -137,23 +147,23 @@ def clean_html(html): return bleach.clean(html, tags=ALLOWED_TAGS) -@app.template_filter() -def quote_plus(t): - return urllib.parse.quote_plus(t) +@app.template_filter() +def quote_plus(t): + return urllib.parse.quote_plus(t) -@app.template_filter() -def is_from_outbox(t): +@app.template_filter() +def is_from_outbox(t): return t.startswith(ID) -@app.template_filter() -def clean(html): - return clean_html(html) +@app.template_filter() +def clean(html): + return clean_html(html) -@app.template_filter() -def html2plaintext(body): +@app.template_filter() +def html2plaintext(body): return html2text(body) @@ -166,13 +176,16 @@ def domain(url): def get_actor(url): if not url: return None - print(f'GET_ACTOR {url}') + print(f"GET_ACTOR {url}") return ACTOR_SERVICE.get(url) + @app.template_filter() def format_time(val): if val: - return datetime.strftime(datetime.strptime(val, '%Y-%m-%dT%H:%M:%SZ'), '%B %d, %Y, %H:%M %p') + return datetime.strftime( + datetime.strptime(val, "%Y-%m-%dT%H:%M:%SZ"), "%B %d, %Y, %H:%M %p" + ) return val @@ -180,26 +193,38 @@ def format_time(val): def format_timeago(val): if val: try: - return timeago.format(datetime.strptime(val, '%Y-%m-%dT%H:%M:%SZ'), datetime.utcnow()) - except: - return timeago.format(datetime.strptime(val, '%Y-%m-%dT%H:%M:%S.%fZ'), datetime.utcnow()) - + return timeago.format( + datetime.strptime(val, "%Y-%m-%dT%H:%M:%SZ"), datetime.utcnow() + ) + except Exception: + return timeago.format( + datetime.strptime(val, "%Y-%m-%dT%H:%M:%S.%fZ"), datetime.utcnow() + ) + return val + def _is_img(filename): filename = filename.lower() - if (filename.endswith('.png') or filename.endswith('.jpg') or filename.endswith('.jpeg') or - filename.endswith('.gif') or filename.endswith('.svg')): + if ( + filename.endswith(".png") + or filename.endswith(".jpg") + or filename.endswith(".jpeg") + or filename.endswith(".gif") + or filename.endswith(".svg") + ): return True return False + @app.template_filter() def not_only_imgs(attachment): for a in attachment: - if not _is_img(a['url']): + if not _is_img(a["url"]): return True return False + @app.template_filter() def is_img(filename): return _is_img(filename) @@ -208,28 +233,29 @@ def is_img(filename): def login_required(f): @wraps(f) def decorated_function(*args, **kwargs): - if not session.get('logged_in'): - return redirect(url_for('login', next=request.url)) + if not session.get("logged_in"): + return redirect(url_for("login", next=request.url)) return f(*args, **kwargs) + return decorated_function def _api_required(): - if session.get('logged_in'): - if request.method not in ['GET', 'HEAD']: + if session.get("logged_in"): + if request.method not in ["GET", "HEAD"]: # If a standard API request is made with a "login session", it must havw a CSRF token csrf.protect() return # Token verification - token = request.headers.get('Authorization', '').replace('Bearer ', '') + token = request.headers.get("Authorization", "").replace("Bearer ", "") if not token: # IndieAuth token - token = request.form.get('access_token', '') + token = request.form.get("access_token", "") # Will raise a BadSignature on bad auth payload = JWT.loads(token) - logger.info(f'api call by {payload}') + logger.info(f"api call by {payload}") def api_required(f): @@ -241,31 +267,36 @@ def api_required(f): abort(401) return f(*args, **kwargs) + return decorated_function def jsonify(**data): - if '@context' not in data: - data['@context'] = config.CTX_AS + if "@context" not in data: + data["@context"] = config.CTX_AS return Response( response=json.dumps(data), - headers={'Content-Type': 'application/json' if app.debug else 'application/activity+json'}, + headers={ + "Content-Type": "application/json" + if app.debug + else "application/activity+json" + }, ) def is_api_request(): - h = request.headers.get('Accept') + h = request.headers.get("Accept") if h is None: return False - h = h.split(',')[0] - if h in HEADERS or h == 'application/json': + h = h.split(",")[0] + if h in HEADERS or h == "application/json": return True return False @app.errorhandler(ValueError) def handle_value_error(error): - logger.error(f'caught value error: {error!r}') + logger.error(f"caught value error: {error!r}") response = flask_jsonify(message=error.args[0]) response.status_code = 400 return response @@ -273,110 +304,114 @@ def handle_value_error(error): @app.errorhandler(Error) def handle_activitypub_error(error): - logger.error(f'caught activitypub error {error!r}') + logger.error(f"caught activitypub error {error!r}") response = flask_jsonify(error.to_dict()) response.status_code = error.status_code return response -# App routes +# App routes ####### # Login -@app.route('/logout') + +@app.route("/logout") @login_required def logout(): - session['logged_in'] = False - return redirect('/') + session["logged_in"] = False + return redirect("/") -@app.route('/login', methods=['POST', 'GET']) +@app.route("/login", methods=["POST", "GET"]) def login(): - devices = [doc['device'] for doc in DB.u2f.find()] + devices = [doc["device"] for doc in DB.u2f.find()] u2f_enabled = True if devices else False - if request.method == 'POST': + if request.method == "POST": csrf.protect() - pwd = request.form.get('pass') + pwd = request.form.get("pass") if pwd and verify_pass(pwd): if devices: - resp = json.loads(request.form.get('resp')) + resp = json.loads(request.form.get("resp")) print(resp) try: - u2f.complete_authentication(session['challenge'], resp) + u2f.complete_authentication(session["challenge"], resp) except ValueError as exc: - print('failed', exc) + print("failed", exc) abort(401) return finally: - session['challenge'] = None + session["challenge"] = None - session['logged_in'] = True - return redirect(request.args.get('redirect') or '/admin') + session["logged_in"] = True + return redirect(request.args.get("redirect") or "/admin") else: abort(401) payload = None if devices: payload = u2f.begin_authentication(ID, devices) - session['challenge'] = payload + session["challenge"] = payload return render_template( - 'login.html', - u2f_enabled=u2f_enabled, - me=ME, - payload=payload, + "login.html", u2f_enabled=u2f_enabled, me=ME, payload=payload ) -@app.route('/remote_follow', methods=['GET', 'POST']) +@app.route("/remote_follow", methods=["GET", "POST"]) def remote_follow(): - if request.method == 'GET': - return render_template('remote_follow.html') + if request.method == "GET": + return render_template("remote_follow.html") csrf.protect() - return redirect(get_remote_follow_template('@'+request.form.get('profile')).format(uri=f'{USERNAME}@{DOMAIN}')) + return redirect( + get_remote_follow_template("@" + request.form.get("profile")).format( + uri=f"{USERNAME}@{DOMAIN}" + ) + ) -@app.route('/authorize_follow', methods=['GET', 'POST']) +@app.route("/authorize_follow", methods=["GET", "POST"]) @login_required def authorize_follow(): - if request.method == 'GET': - return render_template('authorize_remote_follow.html', profile=request.args.get('profile')) + if request.method == "GET": + return render_template( + "authorize_remote_follow.html", profile=request.args.get("profile") + ) - actor = get_actor_url(request.form.get('profile')) + actor = get_actor_url(request.form.get("profile")) if not actor: abort(500) - if DB.following.find({'remote_actor': actor}).count() > 0: - return redirect('/following') + if DB.following.find({"remote_actor": actor}).count() > 0: + return redirect("/following") - follow = activitypub.Follow(object=actor) - follow.post_to_outbox() - return redirect('/following') + follow = activitypub.Follow(actor=MY_PERSON.id, object=actor) + OUTBOX.post(follow) + + return redirect("/following") -@app.route('/u2f/register', methods=['GET', 'POST']) +@app.route("/u2f/register", methods=["GET", "POST"]) @login_required def u2f_register(): # TODO(tsileo): ensure no duplicates - if request.method == 'GET': + if request.method == "GET": payload = u2f.begin_registration(ID) - session['challenge'] = payload - return render_template( - 'u2f.html', - payload=payload, - ) + session["challenge"] = payload + return render_template("u2f.html", payload=payload) else: - resp = json.loads(request.form.get('resp')) - device, device_cert = u2f.complete_registration(session['challenge'], resp) - session['challenge'] = None - DB.u2f.insert_one({'device': device, 'cert': device_cert}) - return '' + resp = json.loads(request.form.get("resp")) + device, device_cert = u2f.complete_registration(session["challenge"], resp) + session["challenge"] = None + DB.u2f.insert_one({"device": device, "cert": device_cert}) + return "" + ####### # Activity pub routes -@app.route('/') + +@app.route("/") def index(): if is_api_request(): return jsonify(**ME) @@ -384,31 +419,41 @@ def index(): # FIXME(tsileo): implements pagination, also for the followers/following page limit = 50 q = { - 'type': 'Create', - 'activity.object.type': 'Note', - 'activity.object.inReplyTo': None, - 'meta.deleted': False, + "type": "Create", + "activity.object.type": "Note", + "activity.object.inReplyTo": None, + "meta.deleted": False, } - c = request.args.get('cursor') + c = request.args.get("cursor") if c: - q['_id'] = {'$lt': ObjectId(c)} + q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list(DB.outbox.find({'$or': [q, {'type': 'Announce', 'meta.undo': False}]}, limit=limit).sort('_id', -1)) + outbox_data = list( + DB.outbox.find( + {"$or": [q, {"type": "Announce", "meta.undo": False}]}, limit=limit + ).sort("_id", -1) + ) cursor = None if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]['_id']) + cursor = str(outbox_data[-1]["_id"]) for data in outbox_data: - if data['type'] == 'Announce': + if data["type"] == "Announce": print(data) - if data['activity']['object'].startswith('http'): - data['ref'] = {'activity': {'object': OBJECT_SERVICE.get(data['activity']['object'])}, 'meta': {}} - + if data["activity"]["object"].startswith("http"): + data["ref"] = { + "activity": { + "object": OBJECT_SERVICE.get(data["activity"]["object"]) + }, + "meta": {}, + } return render_template( - 'index.html', + "index.html", me=ME, - notes=DB.inbox.find({'type': 'Create', 'activity.object.type': 'Note', 'meta.deleted': False}).count(), + notes=DB.inbox.find( + {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} + ).count(), followers=DB.followers.count(), following=DB.following.count(), outbox_data=outbox_data, @@ -416,34 +461,40 @@ def index(): ) -@app.route('/with_replies') +@app.route("/with_replies") def with_replies(): limit = 50 - q = { - 'type': 'Create', - 'activity.object.type': 'Note', - 'meta.deleted': False, - } - c = request.args.get('cursor') + q = {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} + c = request.args.get("cursor") if c: - q['_id'] = {'$lt': ObjectId(c)} + q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list(DB.outbox.find({'$or': [q, {'type': 'Announce', 'meta.undo': False}]}, limit=limit).sort('_id', -1)) + outbox_data = list( + DB.outbox.find( + {"$or": [q, {"type": "Announce", "meta.undo": False}]}, limit=limit + ).sort("_id", -1) + ) cursor = None if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]['_id']) + cursor = str(outbox_data[-1]["_id"]) for data in outbox_data: - if data['type'] == 'Announce': + if data["type"] == "Announce": print(data) - if data['activity']['object'].startswith('http'): - data['ref'] = {'activity': {'object': OBJECT_SERVICE.get(data['activity']['object'])}, 'meta': {}} - + if data["activity"]["object"].startswith("http"): + data["ref"] = { + "activity": { + "object": OBJECT_SERVICE.get(data["activity"]["object"]) + }, + "meta": {}, + } return render_template( - 'index.html', + "index.html", me=ME, - notes=DB.inbox.find({'type': 'Create', 'activity.object.type': 'Note', 'meta.deleted': False}).count(), + notes=DB.inbox.find( + {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} + ).count(), followers=DB.followers.count(), following=DB.following.count(), outbox_data=outbox_data, @@ -452,17 +503,17 @@ def with_replies(): def _build_thread(data, include_children=True): - data['_requested'] = True - root_id = data['meta'].get('thread_root_parent', data['activity']['object']['id']) + data["_requested"] = True + root_id = data["meta"].get("thread_root_parent", data["activity"]["object"]["id"]) - thread_ids = data['meta'].get('thread_parents', []) + thread_ids = data["meta"].get("thread_parents", []) if include_children: - thread_ids.extend(data['meta'].get('thread_children', [])) + thread_ids.extend(data["meta"].get("thread_children", [])) query = { - 'activity.object.id': {'$in': thread_ids}, - 'type': 'Create', - 'meta.deleted': False, # TODO(tsileo): handle Tombstone instead of filtering them + "activity.object.id": {"$in": thread_ids}, + "type": "Create", + "meta.deleted": False, # TODO(tsileo): handle Tombstone instead of filtering them } # Fetch the root replies, and the children replies = [data] + list(DB.inbox.find(query)) + list(DB.outbox.find(query)) @@ -470,385 +521,427 @@ def _build_thread(data, include_children=True): # Index all the IDs in order to build a tree idx = {} for rep in replies: - rep_id = rep['activity']['object']['id'] + rep_id = rep["activity"]["object"]["id"] idx[rep_id] = rep.copy() - idx[rep_id]['_nodes'] = [] + idx[rep_id]["_nodes"] = [] # Build the tree for rep in replies: - rep_id = rep['activity']['object']['id'] + rep_id = rep["activity"]["object"]["id"] if rep_id == root_id: continue - reply_of = rep['activity']['object']['inReplyTo'] - idx[reply_of]['_nodes'].append(rep) + reply_of = rep["activity"]["object"]["inReplyTo"] + idx[reply_of]["_nodes"].append(rep) # Flatten the tree thread = [] + def _flatten(node, level=0): - node['_level'] = level + node["_level"] = level thread.append(node) - - for snode in sorted(idx[node['activity']['object']['id']]['_nodes'], key=lambda d: d['activity']['object']['published']): - _flatten(snode, level=level+1) + + for snode in sorted( + idx[node["activity"]["object"]["id"]]["_nodes"], + key=lambda d: d["activity"]["object"]["published"], + ): + _flatten(snode, level=level + 1) + _flatten(idx[root_id]) return thread -@app.route('/note/') -def note_by_id(note_id): - data = DB.outbox.find_one({'id': note_id}) - if not data: +@app.route("/note/") +def note_by_id(note_id): + data = DB.outbox.find_one({"remote_id": back.activity_url(note_id)}) + if not data: abort(404) - if data['meta'].get('deleted', False): + if data["meta"].get("deleted", False): abort(410) thread = _build_thread(data) + likes = list( + DB.inbox.find( + { + "meta.undo": False, + "type": ActivityType.LIKE.value, + "$or": [ + {"activity.object.id": data["activity"]["object"]["id"]}, + {"activity.object": data["activity"]["object"]["id"]}, + ], + } + ) + ) + likes = [ACTOR_SERVICE.get(doc["activity"]["actor"]) for doc in likes] - likes = list(DB.inbox.find({ - 'meta.undo': False, - 'type': ActivityType.LIKE.value, - '$or': [{'activity.object.id': data['activity']['object']['id']}, - {'activity.object': data['activity']['object']['id']}], - })) - likes = [ACTOR_SERVICE.get(doc['activity']['actor']) for doc in likes] + shares = list( + DB.inbox.find( + { + "meta.undo": False, + "type": ActivityType.ANNOUNCE.value, + "$or": [ + {"activity.object.id": data["activity"]["object"]["id"]}, + {"activity.object": data["activity"]["object"]["id"]}, + ], + } + ) + ) + shares = [ACTOR_SERVICE.get(doc["activity"]["actor"]) for doc in shares] - shares = list(DB.inbox.find({ - 'meta.undo': False, - 'type': ActivityType.ANNOUNCE.value, - '$or': [{'activity.object.id': data['activity']['object']['id']}, - {'activity.object': data['activity']['object']['id']}], - })) - shares = [ACTOR_SERVICE.get(doc['activity']['actor']) for doc in shares] - - return render_template('note.html', likes=likes, shares=shares, me=ME, thread=thread, note=data) - - -@app.route('/nodeinfo') -def nodeinfo(): - return Response( - headers={'Content-Type': 'application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#'}, - response=json.dumps({ - 'version': '2.0', - 'software': {'name': 'microblogpub', 'version': f'Microblog.pub {VERSION}'}, - 'protocols': ['activitypub'], - 'services': {'inbound': [], 'outbound': []}, - 'openRegistrations': False, - 'usage': {'users': {'total': 1}, 'localPosts': DB.outbox.count()}, - 'metadata': { - 'sourceCode': 'https://github.com/tsileo/microblog.pub', - 'nodeName': f'@{USERNAME}@{DOMAIN}', - }, - }), + return render_template( + "note.html", likes=likes, shares=shares, me=ME, thread=thread, note=data ) -@app.route('/.well-known/nodeinfo') +@app.route("/nodeinfo") +def nodeinfo(): + return Response( + headers={ + "Content-Type": "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#" + }, + response=json.dumps( + { + "version": "2.0", + "software": { + "name": "microblogpub", + "version": f"Microblog.pub {VERSION}", + }, + "protocols": ["activitypub"], + "services": {"inbound": [], "outbound": []}, + "openRegistrations": False, + "usage": {"users": {"total": 1}, "localPosts": DB.outbox.count()}, + "metadata": { + "sourceCode": "https://github.com/tsileo/microblog.pub", + "nodeName": f"@{USERNAME}@{DOMAIN}", + }, + } + ), + ) + + +@app.route("/.well-known/nodeinfo") def wellknown_nodeinfo(): return flask_jsonify( links=[ { - 'rel': 'http://nodeinfo.diaspora.software/ns/schema/2.0', - 'href': f'{ID}/nodeinfo', + "rel": "http://nodeinfo.diaspora.software/ns/schema/2.0", + "href": f"{ID}/nodeinfo", } - - ], + ] ) -@app.route('/.well-known/webfinger') +@app.route("/.well-known/webfinger") def wellknown_webfinger(): """Enable WebFinger support, required for Mastodon interopability.""" - resource = request.args.get('resource') - if resource not in [f'acct:{USERNAME}@{DOMAIN}', ID]: + resource = request.args.get("resource") + if resource not in [f"acct:{USERNAME}@{DOMAIN}", ID]: abort(404) out = { - "subject": f'acct:{USERNAME}@{DOMAIN}', + "subject": f"acct:{USERNAME}@{DOMAIN}", "aliases": [ID], "links": [ - {"rel": "http://webfinger.net/rel/profile-page", "type": "text/html", "href": BASE_URL}, + { + "rel": "http://webfinger.net/rel/profile-page", + "type": "text/html", + "href": BASE_URL, + }, {"rel": "self", "type": "application/activity+json", "href": ID}, - {"rel":"http://ostatus.org/schema/1.0/subscribe", "template": BASE_URL+"/authorize_follow?profile={uri}"}, + { + "rel": "http://ostatus.org/schema/1.0/subscribe", + "template": BASE_URL + "/authorize_follow?profile={uri}", + }, ], } return Response( response=json.dumps(out), - headers={'Content-Type': 'application/jrd+json; charset=utf-8' if not app.debug else 'application/json'}, + headers={ + "Content-Type": "application/jrd+json; charset=utf-8" + if not app.debug + else "application/json" + }, ) def add_extra_collection(raw_doc: Dict[str, Any]) -> Dict[str, Any]: - if raw_doc['activity']['type'] != ActivityType.CREATE.value: + if raw_doc["activity"]["type"] != ActivityType.CREATE.value: return raw_doc - raw_doc['activity']['object']['replies'] = embed_collection( - raw_doc.get('meta', {}).get('count_direct_reply', 0), - f'{ID}/outbox/{raw_doc["id"]}/replies', + raw_doc["activity"]["object"]["replies"] = embed_collection( + raw_doc.get("meta", {}).get("count_direct_reply", 0), + f'{raw_doc["remote_id"]}/replies', ) - raw_doc['activity']['object']['likes'] = embed_collection( - raw_doc.get('meta', {}).get('count_like', 0), - f'{ID}/outbox/{raw_doc["id"]}/likes', + raw_doc["activity"]["object"]["likes"] = embed_collection( + raw_doc.get("meta", {}).get("count_like", 0), f'{raw_doc["remote_id"]}/likes' ) - raw_doc['activity']['object']['shares'] = embed_collection( - raw_doc.get('meta', {}).get('count_boost', 0), - f'{ID}/outbox/{raw_doc["id"]}/shares', + raw_doc["activity"]["object"]["shares"] = embed_collection( + raw_doc.get("meta", {}).get("count_boost", 0), f'{raw_doc["remote_id"]}/shares' ) return raw_doc def remove_context(activity: Dict[str, Any]) -> Dict[str, Any]: - if '@context' in activity: - del activity['@context'] + if "@context" in activity: + del activity["@context"] return activity def activity_from_doc(raw_doc: Dict[str, Any], embed: bool = False) -> Dict[str, Any]: raw_doc = add_extra_collection(raw_doc) - activity = clean_activity(raw_doc['activity']) + activity = clean_activity(raw_doc["activity"]) if embed: return remove_context(activity) return activity - -@app.route('/outbox', methods=['GET', 'POST']) -def outbox(): - if request.method == 'GET': - if not is_api_request(): - abort(404) +@app.route("/outbox", methods=["GET", "POST"]) +def outbox(): + if request.method == "GET": + if not is_api_request(): + abort(404) # TODO(tsileo): filter the outbox if not authenticated # FIXME(tsileo): filter deleted, add query support for build_ordered_collection q = { - 'meta.deleted': False, - #'type': {'$in': [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, + "meta.deleted": False, + # 'type': {'$in': [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, } - return jsonify(**activitypub.build_ordered_collection( - DB.outbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: activity_from_doc(doc, embed=True), - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.outbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: activity_from_doc(doc, embed=True), + ) + ) # Handle POST request try: _api_required() except BadSignature: abort(401) - + data = request.get_json(force=True) print(data) - activity = activitypub.parse_activity(data) - - if activity.type_enum == ActivityType.NOTE: - activity = activity.build_create() - - activity.post_to_outbox() + activity = ap.parse_activity(data) + OUTBOX.post(activity) # Purge the cache if a custom hook is set, as new content was published custom_cache_purge_hook() - return Response(status=201, headers={'Location': activity.id}) + return Response(status=201, headers={"Location": activity.id}) -@app.route('/outbox/') +@app.route("/outbox/") def outbox_detail(item_id): - doc = DB.outbox.find_one({'id': item_id}) - if doc['meta'].get('deleted', False): - obj = activitypub.parse_activity(doc['activity']) + doc = DB.outbox.find_one({"remote_id": back.activity_url(item_id)}) + if doc["meta"].get("deleted", False): + obj = ap.parse_activity(doc["activity"]) resp = jsonify(**obj.get_object().get_tombstone()) resp.status_code = 410 return resp return jsonify(**activity_from_doc(doc)) -@app.route('/outbox//activity') +@app.route("/outbox//activity") def outbox_activity(item_id): # TODO(tsileo): handle Tombstone - data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + data = DB.outbox.find_one( + {"remote_id": back.activity_url(item_id), "meta.deleted": False} + ) if not data: abort(404) obj = activity_from_doc(data) - if obj['type'] != ActivityType.CREATE.value: + if obj["type"] != ActivityType.CREATE.value: abort(404) - return jsonify(**obj['object']) + return jsonify(**obj["object"]) -@app.route('/outbox//replies') +@app.route("/outbox//replies") def outbox_activity_replies(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + data = DB.outbox.find_one( + {"remote_id": back.activity_url(item_id), "meta.deleted": False} + ) if not data: abort(404) - obj = activitypub.parse_activity(data['activity']) - if obj.type_enum != ActivityType.CREATE: + obj = ap.parse_activity(data["activity"]) + if obj.ACTIVITY_TYPE != ActivityType.CREATE: abort(404) q = { - 'meta.deleted': False, - 'type': ActivityType.CREATE.value, - 'activity.object.inReplyTo': obj.get_object().id, + "meta.deleted": False, + "type": ActivityType.CREATE.value, + "activity.object.inReplyTo": obj.get_object().id, } - return jsonify(**activitypub.build_ordered_collection( - DB.inbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: doc['activity']['object'], - col_name=f'outbox/{item_id}/replies', - first_page=request.args.get('page') == 'first', - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.inbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: doc["activity"]["object"], + col_name=f"outbox/{item_id}/replies", + first_page=request.args.get("page") == "first", + ) + ) -@app.route('/outbox//likes') +@app.route("/outbox//likes") def outbox_activity_likes(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + data = DB.outbox.find_one( + {"remote_id": back.activity_url(item_id), "meta.deleted": False} + ) if not data: abort(404) - obj = activitypub.parse_activity(data['activity']) - if obj.type_enum != ActivityType.CREATE: + obj = ap.parse_activity(data["activity"]) + if obj.ACTIVITY_TYPE != ActivityType.CREATE: abort(404) q = { - 'meta.undo': False, - 'type': ActivityType.LIKE.value, - '$or': [{'activity.object.id': obj.get_object().id}, - {'activity.object': obj.get_object().id}], + "meta.undo": False, + "type": ActivityType.LIKE.value, + "$or": [ + {"activity.object.id": obj.get_object().id}, + {"activity.object": obj.get_object().id}, + ], } - return jsonify(**activitypub.build_ordered_collection( - DB.inbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: remove_context(doc['activity']), - col_name=f'outbox/{item_id}/likes', - first_page=request.args.get('page') == 'first', - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.inbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: remove_context(doc["activity"]), + col_name=f"outbox/{item_id}/likes", + first_page=request.args.get("page") == "first", + ) + ) -@app.route('/outbox//shares') +@app.route("/outbox//shares") def outbox_activity_shares(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one({'id': item_id, 'meta.deleted': False}) + data = DB.outbox.find_one( + {"remote_id": back.activity_url(item_id), "meta.deleted": False} + ) if not data: abort(404) - obj = activitypub.parse_activity(data['activity']) - if obj.type_enum != ActivityType.CREATE: + obj = ap.parse_activity(data["activity"]) + if obj.ACTIVITY_TYPE != ActivityType.CREATE: abort(404) q = { - 'meta.undo': False, - 'type': ActivityType.ANNOUNCE.value, - '$or': [{'activity.object.id': obj.get_object().id}, - {'activity.object': obj.get_object().id}], + "meta.undo": False, + "type": ActivityType.ANNOUNCE.value, + "$or": [ + {"activity.object.id": obj.get_object().id}, + {"activity.object": obj.get_object().id}, + ], } - return jsonify(**activitypub.build_ordered_collection( - DB.inbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: remove_context(doc['activity']), - col_name=f'outbox/{item_id}/shares', - first_page=request.args.get('page') == 'first', - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.inbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: remove_context(doc["activity"]), + col_name=f"outbox/{item_id}/shares", + first_page=request.args.get("page") == "first", + ) + ) -@app.route('/admin', methods=['GET']) +@app.route("/admin", methods=["GET"]) @login_required def admin(): - q = { - 'meta.deleted': False, - 'meta.undo': False, - 'type': ActivityType.LIKE.value, - } + q = {"meta.deleted": False, "meta.undo": False, "type": ActivityType.LIKE.value} col_liked = DB.outbox.count(q) return render_template( - 'admin.html', - instances=list(DB.instances.find()), - inbox_size=DB.inbox.count(), - outbox_size=DB.outbox.count(), - object_cache_size=DB.objects_cache.count(), - actor_cache_size=DB.actors_cache.count(), - col_liked=col_liked, - col_followers=DB.followers.count(), - col_following=DB.following.count(), + "admin.html", + instances=list(DB.instances.find()), + inbox_size=DB.inbox.count(), + outbox_size=DB.outbox.count(), + object_cache_size=DB.objects_cache.count(), + actor_cache_size=DB.actors_cache.count(), + col_liked=col_liked, + col_followers=DB.followers.count(), + col_following=DB.following.count(), ) - -@app.route('/new', methods=['GET']) + +@app.route("/new", methods=["GET"]) @login_required def new(): reply_id = None - content = '' + content = "" thread = [] - if request.args.get('reply'): - data = DB.inbox.find_one({'activity.object.id': request.args.get('reply')}) + if request.args.get("reply"): + data = DB.inbox.find_one({"activity.object.id": request.args.get("reply")}) if not data: - data = DB.outbox.find_one({'activity.object.id': request.args.get('reply')}) + data = DB.outbox.find_one({"activity.object.id": request.args.get("reply")}) if not data: abort(400) - reply = activitypub.parse_activity(data['activity']) + reply = ap.parse_activity(data["activity"]) reply_id = reply.id - if reply.type_enum == ActivityType.CREATE: + if reply.ACTIVITY_TYPE == ActivityType.CREATE: reply_id = reply.get_object().id actor = reply.get_actor() domain = urlparse(actor.id).netloc # FIXME(tsileo): if reply of reply, fetch all participants - content = f'@{actor.preferredUsername}@{domain} ' - thread = _build_thread( - data, - include_children=False, - ) + content = f"@{actor.preferredUsername}@{domain} " + thread = _build_thread(data, include_children=False) - return render_template( - 'new.html', - reply=reply_id, - content=content, - thread=thread, - ) + return render_template("new.html", reply=reply_id, content=content, thread=thread) -@app.route('/notifications') +@app.route("/notifications") @login_required def notifications(): # FIXME(tsileo): implements pagination, also for the followers/following page limit = 50 q = { - 'type': 'Create', - 'activity.object.tag.type': 'Mention', - 'activity.object.tag.name': f'@{USERNAME}@{DOMAIN}', - 'meta.deleted': False, + "type": "Create", + "activity.object.tag.type": "Mention", + "activity.object.tag.name": f"@{USERNAME}@{DOMAIN}", + "meta.deleted": False, } # TODO(tsileo): also include replies via regex on Create replyTo - q = {'$or': [q, {'type': 'Follow'}, {'type': 'Accept'}, {'type': 'Undo', 'activity.object.type': 'Follow'}, - {'type': 'Announce', 'activity.object': {'$regex': f'^{BASE_URL}'}}, - {'type': 'Create', 'activity.object.inReplyTo': {'$regex': f'^{BASE_URL}'}}, - ]} + q = { + "$or": [ + q, + {"type": "Follow"}, + {"type": "Accept"}, + {"type": "Undo", "activity.object.type": "Follow"}, + {"type": "Announce", "activity.object": {"$regex": f"^{BASE_URL}"}}, + {"type": "Create", "activity.object.inReplyTo": {"$regex": f"^{BASE_URL}"}}, + ] + } print(q) - c = request.args.get('cursor') + c = request.args.get("cursor") if c: - q['_id'] = {'$lt': ObjectId(c)} + q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list(DB.inbox.find(q, limit=limit).sort('_id', -1)) + outbox_data = list(DB.inbox.find(q, limit=limit).sort("_id", -1)) cursor = None if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]['_id']) + cursor = str(outbox_data[-1]["_id"]) # TODO(tsileo): fix the annonce handling, copy it from /stream - #for data in outbox_data: + # for data in outbox_data: # if data['type'] == 'Announce': # print(data) # if data['activity']['object'].startswith('http') and data['activity']['object'] in objcache: @@ -857,14 +950,10 @@ def notifications(): # else: # out.append(data) - return render_template( - 'stream.html', - inbox_data=outbox_data, - cursor=cursor, - ) + return render_template("stream.html", inbox_data=outbox_data, cursor=cursor) -@app.route('/api/key') +@app.route("/api/key") @login_required def api_user_key(): return flask_jsonify(api_key=ADMIN_API_KEY) @@ -878,25 +967,27 @@ def _user_api_arg(key: str, **kwargs): oid = request.args.get(key) or request.form.get(key) if not oid: - if 'default' in kwargs: - return kwargs.get('default') + if "default" in kwargs: + return kwargs.get("default") - raise ValueError(f'missing {key}') + raise ValueError(f"missing {key}") return oid def _user_api_get_note(from_outbox: bool = False): - oid = _user_api_arg('id') - note = activitypub.parse_activity(OBJECT_SERVICE.get(oid), expected=ActivityType.NOTE) + oid = _user_api_arg("id") + note = ap.parse_activity(OBJECT_SERVICE.get(oid), expected=ActivityType.NOTE) if from_outbox and not note.id.startswith(ID): - raise NotFromOutboxError(f'cannot delete {note.id}, id must be owned by the server') + raise NotFromOutboxError( + f"cannot load {note.id}, id must be owned by the server" + ) return note def _user_api_response(**kwargs): - _redirect = _user_api_arg('redirect', default=None) + _redirect = _user_api_arg("redirect", default=None) if _redirect: return redirect(_redirect) @@ -905,397 +996,430 @@ def _user_api_response(**kwargs): return resp -@app.route('/api/note/delete', methods=['POST']) +@app.route("/api/note/delete", methods=["POST"]) @api_required def api_delete(): """API endpoint to delete a Note activity.""" note = _user_api_get_note(from_outbox=True) delete = note.build_delete() - delete.post_to_outbox() + OUTBOX.post(delete) return _user_api_response(activity=delete.id) -@app.route('/api/boost', methods=['POST']) +@app.route("/api/boost", methods=["POST"]) @api_required def api_boost(): note = _user_api_get_note() - announce = note.build_announce() - announce.post_to_outbox() + announce = note.build_announce(MY_PERSON) + OUTBOX.post(announce) return _user_api_response(activity=announce.id) -@app.route('/api/like', methods=['POST']) +@app.route("/api/like", methods=["POST"]) @api_required def api_like(): note = _user_api_get_note() - like = note.build_like() - like.post_to_outbox() + like = note.build_like(MY_PERSON) + OUTBOX.post(like) return _user_api_response(activity=like.id) -@app.route('/api/undo', methods=['POST']) +@app.route("/api/undo", methods=["POST"]) @api_required def api_undo(): - oid = _user_api_arg('id') - doc = DB.outbox.find_one({'$or': [{'id': oid}, {'remote_id': oid}]}) + oid = _user_api_arg("id") + doc = DB.outbox.find_one( + {"$or": [{"remote_id": back.activity_url(oid)}, {"remote_id": oid}]} + ) if not doc: - raise ActivityNotFoundError(f'cannot found {oid}') + raise ActivityNotFoundError(f"cannot found {oid}") - obj = activitypub.parse_activity(doc.get('activity')) + obj = ap.parse_activity(doc.get("activity")) # FIXME(tsileo): detect already undo-ed and make this API call idempotent undo = obj.build_undo() - undo.post_to_outbox() + OUTBOX.post(undo) return _user_api_response(activity=undo.id) -@app.route('/stream') +@app.route("/stream") @login_required def stream(): # FIXME(tsileo): implements pagination, also for the followers/following page limit = 100 q = { - 'type': 'Create', - 'activity.object.type': 'Note', - 'activity.object.inReplyTo': None, - 'meta.deleted': False, + "type": "Create", + "activity.object.type": "Note", + "activity.object.inReplyTo": None, + "meta.deleted": False, } - c = request.args.get('cursor') + c = request.args.get("cursor") if c: - q['_id'] = {'$lt': ObjectId(c)} + q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list(DB.inbox.find( - { - '$or': [ - q, - { - 'type': 'Announce', - }, - ] - }, limit=limit).sort('activity.published', -1)) + outbox_data = list( + DB.inbox.find({"$or": [q, {"type": "Announce"}]}, limit=limit).sort( + "activity.published", -1 + ) + ) cursor = None if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]['_id']) + cursor = str(outbox_data[-1]["_id"]) out = [] objcache = {} - cached = list(DB.objects_cache.find({'meta.part_of_stream': True}, limit=limit*3).sort('meta.announce_published', -1)) + cached = list( + DB.objects_cache.find({"meta.part_of_stream": True}, limit=limit * 3).sort( + "meta.announce_published", -1 + ) + ) for c in cached: - objcache[c['object_id']] = c['cached_object'] + objcache[c["object_id"]] = c["cached_object"] for data in outbox_data: - if data['type'] == 'Announce': - if data['activity']['object'].startswith('http') and data['activity']['object'] in objcache: - data['ref'] = {'activity': {'object': objcache[data['activity']['object']]}, 'meta': {}} + if data["type"] == "Announce": + if ( + data["activity"]["object"].startswith("http") + and data["activity"]["object"] in objcache + ): + data["ref"] = { + "activity": {"object": objcache[data["activity"]["object"]]}, + "meta": {}, + } out.append(data) else: - print('OMG', data) + print("OMG", data) else: out.append(data) - return render_template( - 'stream.html', - inbox_data=out, - cursor=cursor, - ) + return render_template("stream.html", inbox_data=out, cursor=cursor) -@app.route('/inbox', methods=['GET', 'POST']) -def inbox(): - if request.method == 'GET': - if not is_api_request(): - abort(404) +@app.route("/inbox", methods=["GET", "POST"]) +def inbox(): + if request.method == "GET": + if not is_api_request(): + abort(404) try: _api_required() except BadSignature: abort(404) - return jsonify(**activitypub.build_ordered_collection( - DB.inbox, - q={'meta.deleted': False}, - cursor=request.args.get('cursor'), - map_func=lambda doc: remove_context(doc['activity']), - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.inbox, + q={"meta.deleted": False}, + cursor=request.args.get("cursor"), + map_func=lambda doc: remove_context(doc["activity"]), + ) + ) - data = request.get_json(force=True) - logger.debug(f'req_headers={request.headers}') - logger.debug(f'raw_data={data}') - try: + data = request.get_json(force=True) + logger.debug(f"req_headers={request.headers}") + logger.debug(f"raw_data={data}") + """try: if not verify_request(ACTOR_SERVICE): - raise Exception('failed to verify request') + raise Exception("failed to verify request") except Exception: - logger.exception('failed to verify request, trying to verify the payload by fetching the remote') + logger.exception( + "failed to verify request, trying to verify the payload by fetching the remote" + ) try: - data = OBJECT_SERVICE.get(data['id']) + data = OBJECT_SERVICE.get(data["id"]) except Exception: logger.exception(f'failed to fetch remote id at {data["id"]}') return Response( status=422, - headers={'Content-Type': 'application/json'}, - response=json.dumps({'error': 'failed to verify request (using HTTP signatures or fetching the IRI)'}), + headers={"Content-Type": "application/json"}, + response=json.dumps( + { + "error": "failed to verify request (using HTTP signatures or fetching the IRI)" + } + ), ) + """ + activity = ap.parse_activity(data) + logger.debug(f"inbox activity={activity}/{data}") + INBOX.post(activity) - activity = activitypub.parse_activity(data) - logger.debug(f'inbox activity={activity}/{data}') - activity.process_from_inbox() - - return Response( - status=201, - ) + return Response(status=201) -@app.route('/api/debug', methods=['GET', 'DELETE']) +def without_id(l): + out = [] + for d in l: + if "_id" in d: + del d["_id"] + out.append(d) + return out + + +@app.route("/api/debug", methods=["GET", "DELETE"]) @api_required def api_debug(): """Endpoint used/needed for testing, only works in DEBUG_MODE.""" if not DEBUG_MODE: - return flask_jsonify(message='DEBUG_MODE is off') + return flask_jsonify(message="DEBUG_MODE is off") - if request.method == 'DELETE': + if request.method == "DELETE": _drop_db() - return flask_jsonify(message='DB dropped') + return flask_jsonify(message="DB dropped") return flask_jsonify( inbox=DB.inbox.count(), outbox=DB.outbox.count(), + outbox_data=without_id(DB.outbox.find()), ) -@app.route('/api/upload', methods=['POST']) +@app.route("/api/upload", methods=["POST"]) @api_required def api_upload(): - file = request.files['file'] + file = request.files["file"] rfilename = secure_filename(file.filename) prefix = hashlib.sha256(os.urandom(32)).hexdigest()[:6] mtype = mimetypes.guess_type(rfilename)[0] - filename = f'{prefix}_{rfilename}' - file.save(os.path.join('static', 'media', filename)) + filename = f"{prefix}_{rfilename}" + file.save(os.path.join("static", "media", filename)) # Remove EXIF metadata - if filename.lower().endswith('.jpg') or filename.lower().endswith('.jpeg'): - piexif.remove(os.path.join('static', 'media', filename)) + if filename.lower().endswith(".jpg") or filename.lower().endswith(".jpeg"): + piexif.remove(os.path.join("static", "media", filename)) - print('upload OK') + print("upload OK") print(filename) attachment = [ - {'mediaType': mtype, - 'name': rfilename, - 'type': 'Document', - 'url': BASE_URL + f'/static/media/{filename}' - }, + { + "mediaType": mtype, + "name": rfilename, + "type": "Document", + "url": BASE_URL + f"/static/media/{filename}", + } ] print(attachment) - content = request.args.get('content') - to = request.args.get('to') - note = activitypub.Note( - cc=[ID+'/followers'], - to=[to if to else config.AS_PUBLIC], + content = request.args.get("content") + to = request.args.get("to") + note = ap.Note( + attributedTo=MY_PERSON.id, + cc=[ID + "/followers"], + to=[to if to else ap.AS_PUBLIC], content=content, # TODO(tsileo): handle markdown attachment=attachment, ) - print('post_note_init') + print("post_note_init") print(note) create = note.build_create() print(create) print(create.to_dict()) - create.post_to_outbox() - print('posted') - - return Response( - status=201, - response='OK', - ) + OUTBOX.post(create) + print("posted") + + return Response(status=201, response="OK") -@app.route('/api/new_note', methods=['POST']) -@api_required -def api_new_note(): - source = _user_api_arg('content') +@app.route("/api/new_note", methods=["POST"]) +@api_required +def api_new_note(): + source = _user_api_arg("content") if not source: - raise ValueError('missing content') - + raise ValueError("missing content") + _reply, reply = None, None try: - _reply = _user_api_arg('reply') + _reply = _user_api_arg("reply") except ValueError: pass - content, tags = parse_markdown(source) - to = request.args.get('to') - cc = [ID+'/followers'] - + content, tags = parse_markdown(source) + to = request.args.get("to") + cc = [ID + "/followers"] + if _reply: - reply = activitypub.parse_activity(OBJECT_SERVICE.get(_reply)) + reply = ap.parse_activity(OBJECT_SERVICE.get(_reply)) cc.append(reply.attributedTo) for tag in tags: - if tag['type'] == 'Mention': - cc.append(tag['href']) + if tag["type"] == "Mention": + cc.append(tag["href"]) - note = activitypub.Note( - cc=list(set(cc)), - to=[to if to else config.AS_PUBLIC], + note = ap.Note( + attributedTo=MY_PERSON.id, + cc=list(set(cc)), + to=[to if to else ap.AS_PUBLIC], content=content, tag=tags, - source={'mediaType': 'text/markdown', 'content': source}, - inReplyTo=reply.id if reply else None + source={"mediaType": "text/markdown", "content": source}, + inReplyTo=reply.id if reply else None, ) create = note.build_create() - create.post_to_outbox() + OUTBOX.post(create) return _user_api_response(activity=create.id) -@app.route('/api/stream') +@app.route("/api/stream") @api_required def api_stream(): return Response( - response=json.dumps(activitypub.build_inbox_json_feed('/api/stream', request.args.get('cursor'))), - headers={'Content-Type': 'application/json'}, + response=json.dumps( + activitypub.build_inbox_json_feed("/api/stream", request.args.get("cursor")) + ), + headers={"Content-Type": "application/json"}, ) -@app.route('/api/block', methods=['POST']) +@app.route("/api/block", methods=["POST"]) @api_required def api_block(): - actor = _user_api_arg('actor') + actor = _user_api_arg("actor") - existing = DB.outbox.find_one({ - 'type': ActivityType.BLOCK.value, - 'activity.object': actor, - 'meta.undo': False, - }) + existing = DB.outbox.find_one( + {"type": ActivityType.BLOCK.value, "activity.object": actor, "meta.undo": False} + ) if existing: - return _user_api_response(activity=existing['activity']['id']) + return _user_api_response(activity=existing["activity"]["id"]) - block = activitypub.Block(object=actor) - block.post_to_outbox() + block = ap.Block(actor=MY_PERSON.id, object=actor) + OUTBOX.post(block) return _user_api_response(activity=block.id) -@app.route('/api/follow', methods=['POST']) +@app.route("/api/follow", methods=["POST"]) @api_required def api_follow(): - actor = _user_api_arg('actor') + actor = _user_api_arg("actor") - existing = DB.following.find_one({'remote_actor': actor}) + existing = DB.following.find_one({"remote_actor": actor}) if existing: - return _user_api_response(activity=existing['activity']['id']) + return _user_api_response(activity=existing["activity"]["id"]) - follow = activitypub.Follow(object=actor) - follow.post_to_outbox() + follow = ap.Follow(actor=MY_PERSON.id, object=actor) + OUTBOX.post(follow) return _user_api_response(activity=follow.id) -@app.route('/followers') -def followers(): - if is_api_request(): +@app.route("/followers") +def followers(): + if is_api_request(): return jsonify( **activitypub.build_ordered_collection( DB.followers, - cursor=request.args.get('cursor'), - map_func=lambda doc: doc['remote_actor'], + cursor=request.args.get("cursor"), + map_func=lambda doc: doc["remote_actor"], ) - ) + ) - followers = [ACTOR_SERVICE.get(doc['remote_actor']) for doc in DB.followers.find(limit=50)] - return render_template( - 'followers.html', + followers = [ + ACTOR_SERVICE.get(doc["remote_actor"]) for doc in DB.followers.find(limit=50) + ] + return render_template( + "followers.html", me=ME, - notes=DB.inbox.find({'object.object.type': 'Note'}).count(), + notes=DB.inbox.find({"object.object.type": "Note"}).count(), followers=DB.followers.count(), following=DB.following.count(), followers_data=followers, ) -@app.route('/following') +@app.route("/following") def following(): if is_api_request(): return jsonify( **activitypub.build_ordered_collection( DB.following, - cursor=request.args.get('cursor'), - map_func=lambda doc: doc['remote_actor'], - ), + cursor=request.args.get("cursor"), + map_func=lambda doc: doc["remote_actor"], + ) ) - - following = [ACTOR_SERVICE.get(doc['remote_actor']) for doc in DB.following.find(limit=50)] + + following = [ + ACTOR_SERVICE.get(doc["remote_actor"]) for doc in DB.following.find(limit=50) + ] return render_template( - 'following.html', + "following.html", me=ME, - notes=DB.inbox.find({'object.object.type': 'Note'}).count(), + notes=DB.inbox.find({"object.object.type": "Note"}).count(), followers=DB.followers.count(), following=DB.following.count(), following_data=following, ) -@app.route('/tags/') +@app.route("/tags/") def tags(tag): - if not DB.outbox.count({'activity.object.tag.type': 'Hashtag', 'activity.object.tag.name': '#'+tag}): + if not DB.outbox.count( + {"activity.object.tag.type": "Hashtag", "activity.object.tag.name": "#" + tag} + ): abort(404) if not is_api_request(): return render_template( - 'tags.html', + "tags.html", tag=tag, - outbox_data=DB.outbox.find({'type': 'Create', 'activity.object.type': 'Note', 'meta.deleted': False, - 'activity.object.tag.type': 'Hashtag', - 'activity.object.tag.name': '#'+tag}), + outbox_data=DB.outbox.find( + { + "type": "Create", + "activity.object.type": "Note", + "meta.deleted": False, + "activity.object.tag.type": "Hashtag", + "activity.object.tag.name": "#" + tag, + } + ), ) q = { - 'meta.deleted': False, - 'meta.undo': False, - 'type': ActivityType.CREATE.value, - 'activity.object.tag.type': 'Hashtag', - 'activity.object.tag.name': '#'+tag, + "meta.deleted": False, + "meta.undo": False, + "type": ActivityType.CREATE.value, + "activity.object.tag.type": "Hashtag", + "activity.object.tag.name": "#" + tag, } - return jsonify(**activitypub.build_ordered_collection( - DB.outbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: doc['activity']['object']['id'], - col_name=f'tags/{tag}', - )) + return jsonify( + **activitypub.build_ordered_collection( + DB.outbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: doc["activity"]["object"]["id"], + col_name=f"tags/{tag}", + ) + ) -@app.route('/liked') +@app.route("/liked") def liked(): if not is_api_request(): abort(404) - q = { - 'meta.deleted': False, - 'meta.undo': False, - 'type': ActivityType.LIKE.value, - } - return jsonify(**activitypub.build_ordered_collection( - DB.outbox, - q=q, - cursor=request.args.get('cursor'), - map_func=lambda doc: doc['activity']['object'], - col_name='liked', - )) + q = {"meta.deleted": False, "meta.undo": False, "type": ActivityType.LIKE.value} + return jsonify( + **activitypub.build_ordered_collection( + DB.outbox, + q=q, + cursor=request.args.get("cursor"), + map_func=lambda doc: doc["activity"]["object"], + col_name="liked", + ) + ) + ####### # IndieAuth def build_auth_resp(payload): - if request.headers.get('Accept') == 'application/json': + if request.headers.get("Accept") == "application/json": return Response( status=200, - headers={'Content-Type': 'application/json'}, + headers={"Content-Type": "application/json"}, response=json.dumps(payload), ) return Response( status=200, - headers={'Content-Type': 'application/x-www-form-urlencoded'}, + headers={"Content-Type": "application/x-www-form-urlencoded"}, response=urlencode(payload), ) @@ -1308,43 +1432,37 @@ def _get_prop(props, name, default=None): return items return default + def get_client_id_data(url): data = mf2py.parse(url=url) - for item in data['items']: - if 'h-x-app' in item['type'] or 'h-app' in item['type']: - props = item.get('properties', {}) + for item in data["items"]: + if "h-x-app" in item["type"] or "h-app" in item["type"]: + props = item.get("properties", {}) print(props) return dict( - logo=_get_prop(props, 'logo'), - name=_get_prop(props, 'name'), - url=_get_prop(props, 'url'), + logo=_get_prop(props, "logo"), + name=_get_prop(props, "name"), + url=_get_prop(props, "url"), ) - return dict( - logo=None, - name=url, - url=url, - ) + return dict(logo=None, name=url, url=url) -@app.route('/indieauth/flow', methods=['POST']) -@login_required -def indieauth_flow(): - auth = dict( - scope=' '.join(request.form.getlist('scopes')), - me=request.form.get('me'), - client_id=request.form.get('client_id'), - state=request.form.get('state'), - redirect_uri=request.form.get('redirect_uri'), - response_type=request.form.get('response_type'), +@app.route("/indieauth/flow", methods=["POST"]) +@login_required +def indieauth_flow(): + auth = dict( + scope=" ".join(request.form.getlist("scopes")), + me=request.form.get("me"), + client_id=request.form.get("client_id"), + state=request.form.get("state"), + redirect_uri=request.form.get("redirect_uri"), + response_type=request.form.get("response_type"), ) - code = binascii.hexlify(os.urandom(8)).decode('utf-8') - auth.update( - code=code, - verified=False, - ) + code = binascii.hexlify(os.urandom(8)).decode("utf-8") + auth.update(code=code, verified=False) print(auth) - if not auth['redirect_uri']: + if not auth["redirect_uri"]: abort(500) DB.indieauth.insert_one(auth) @@ -1354,23 +1472,23 @@ def indieauth_flow(): return redirect(red) -# @app.route('/indieauth', methods=['GET', 'POST']) -def indieauth_endpoint(): - if request.method == 'GET': - if not session.get('logged_in'): - return redirect(url_for('login', next=request.url)) +# @app.route('/indieauth', methods=['GET', 'POST']) +def indieauth_endpoint(): + if request.method == "GET": + if not session.get("logged_in"): + return redirect(url_for("login", next=request.url)) - me = request.args.get('me') - # FIXME(tsileo): ensure me == ID - client_id = request.args.get('client_id') - redirect_uri = request.args.get('redirect_uri') - state = request.args.get('state', '') - response_type = request.args.get('response_type', 'id') - scope = request.args.get('scope', '').split() + me = request.args.get("me") + # FIXME(tsileo): ensure me == ID + client_id = request.args.get("client_id") + redirect_uri = request.args.get("redirect_uri") + state = request.args.get("state", "") + response_type = request.args.get("response_type", "id") + scope = request.args.get("scope", "").split() - print('STATE', state) + print("STATE", state) return render_template( - 'indieauth_flow.html', + "indieauth_flow.html", client=get_client_id_data(client_id), scopes=scope, redirect_uri=redirect_uri, @@ -1381,14 +1499,18 @@ def indieauth_endpoint(): ) # Auth verification via POST - code = request.form.get('code') - redirect_uri = request.form.get('redirect_uri') - client_id = request.form.get('client_id') + code = request.form.get("code") + redirect_uri = request.form.get("redirect_uri") + client_id = request.form.get("client_id") auth = DB.indieauth.find_one_and_update( - {'code': code, 'redirect_uri': redirect_uri, 'client_id': client_id}, #}, # , 'verified': False}, - {'$set': {'verified': True}}, - sort=[('_id', pymongo.DESCENDING)], + { + "code": code, + "redirect_uri": redirect_uri, + "client_id": client_id, + }, # }, # , 'verified': False}, + {"$set": {"verified": True}}, + sort=[("_id", pymongo.DESCENDING)], ) print(auth) print(code, redirect_uri, client_id) @@ -1397,33 +1519,42 @@ def indieauth_endpoint(): abort(403) return - session['logged_in'] = True - me = auth['me'] - state = auth['state'] - scope = ' '.join(auth['scope']) - print('STATE', state) - return build_auth_resp({'me': me, 'state': state, 'scope': scope}) + session["logged_in"] = True + me = auth["me"] + state = auth["state"] + scope = " ".join(auth["scope"]) + print("STATE", state) + return build_auth_resp({"me": me, "state": state, "scope": scope}) -@app.route('/token', methods=['GET', 'POST']) +@app.route("/token", methods=["GET", "POST"]) def token_endpoint(): - if request.method == 'POST': - code = request.form.get('code') - me = request.form.get('me') - redirect_uri = request.form.get('redirect_uri') - client_id = request.form.get('client_id') + if request.method == "POST": + code = request.form.get("code") + me = request.form.get("me") + redirect_uri = request.form.get("redirect_uri") + client_id = request.form.get("client_id") - auth = DB.indieauth.find_one({'code': code, 'me': me, 'redirect_uri': redirect_uri, 'client_id': client_id}) + auth = DB.indieauth.find_one( + { + "code": code, + "me": me, + "redirect_uri": redirect_uri, + "client_id": client_id, + } + ) if not auth: abort(403) - scope = ' '.join(auth['scope']) - payload = dict(me=me, client_id=client_id, scope=scope, ts=datetime.now().timestamp()) - token = JWT.dumps(payload).decode('utf-8') + scope = " ".join(auth["scope"]) + payload = dict( + me=me, client_id=client_id, scope=scope, ts=datetime.now().timestamp() + ) + token = JWT.dumps(payload).decode("utf-8") - return build_auth_resp({'me': me, 'scope': scope, 'access_token': token}) + return build_auth_resp({"me": me, "scope": scope, "access_token": token}) # Token verification - token = request.headers.get('Authorization').replace('Bearer ', '') + token = request.headers.get("Authorization").replace("Bearer ", "") try: payload = JWT.loads(token) except BadSignature: @@ -1431,8 +1562,10 @@ def token_endpoint(): # TODO(tsileo): handle expiration - return build_auth_resp({ - 'me': payload['me'], - 'scope': payload['scope'], - 'client_id': payload['client_id'], - }) + return build_auth_resp( + { + "me": payload["me"], + "scope": payload["scope"], + "client_id": payload["client_id"], + } + ) diff --git a/config.py b/config.py index 506a4d3..44659c1 100644 --- a/config.py +++ b/config.py @@ -1,15 +1,17 @@ -import subprocess import os -import yaml -from pymongo import MongoClient -import requests -from itsdangerous import JSONWebSignatureSerializer +import subprocess from datetime import datetime -from utils import strtobool -from utils.key import Key, KEY_DIR, get_secret_key -from utils.actor_service import ActorService -from utils.object_service import ObjectService +import requests +import yaml +from itsdangerous import JSONWebSignatureSerializer +from pymongo import MongoClient + +from little_boxes import strtobool +from utils.key import KEY_DIR +from utils.key import get_key +from utils.key import get_secret_key + def noop(): pass @@ -17,103 +19,94 @@ def noop(): CUSTOM_CACHE_HOOKS = False try: - from cache_hooks import purge as custom_cache_purge_hook + from cache_hooks import purge as custom_cache_purge_hook except ModuleNotFoundError: custom_cache_purge_hook = noop -VERSION = subprocess.check_output(['git', 'describe', '--always']).split()[0].decode('utf-8') +VERSION = ( + subprocess.check_output(["git", "describe", "--always"]).split()[0].decode("utf-8") +) -DEBUG_MODE = strtobool(os.getenv('MICROBLOGPUB_DEBUG', 'false')) +DEBUG_MODE = strtobool(os.getenv("MICROBLOGPUB_DEBUG", "false")) -CTX_AS = 'https://www.w3.org/ns/activitystreams' -CTX_SECURITY = 'https://w3id.org/security/v1' -AS_PUBLIC = 'https://www.w3.org/ns/activitystreams#Public' +CTX_AS = "https://www.w3.org/ns/activitystreams" +CTX_SECURITY = "https://w3id.org/security/v1" +AS_PUBLIC = "https://www.w3.org/ns/activitystreams#Public" HEADERS = [ - 'application/activity+json', - 'application/ld+json;profile=https://www.w3.org/ns/activitystreams', + "application/activity+json", + "application/ld+json;profile=https://www.w3.org/ns/activitystreams", 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', - 'application/ld+json', + "application/ld+json", ] -with open(os.path.join(KEY_DIR, 'me.yml')) as f: +with open(os.path.join(KEY_DIR, "me.yml")) as f: conf = yaml.load(f) - USERNAME = conf['username'] - NAME = conf['name'] - DOMAIN = conf['domain'] - SCHEME = 'https' if conf.get('https', True) else 'http' - BASE_URL = SCHEME + '://' + DOMAIN + USERNAME = conf["username"] + NAME = conf["name"] + DOMAIN = conf["domain"] + SCHEME = "https" if conf.get("https", True) else "http" + BASE_URL = SCHEME + "://" + DOMAIN ID = BASE_URL - SUMMARY = conf['summary'] - ICON_URL = conf['icon_url'] - PASS = conf['pass'] - PUBLIC_INSTANCES = conf.get('public_instances', []) + SUMMARY = conf["summary"] + ICON_URL = conf["icon_url"] + PASS = conf["pass"] + PUBLIC_INSTANCES = conf.get("public_instances", []) # TODO(tsileo): choose dark/light style - THEME_COLOR = conf.get('theme_color') + THEME_COLOR = conf.get("theme_color") USER_AGENT = ( - f'{requests.utils.default_user_agent()} ' - f'(microblog.pub/{VERSION}; +{BASE_URL})' + f"{requests.utils.default_user_agent()} " f"(microblog.pub/{VERSION}; +{BASE_URL})" ) -# TODO(tsileo): use 'mongo:27017; -# mongo_client = MongoClient(host=['mongo:27017']) mongo_client = MongoClient( - host=[os.getenv('MICROBLOGPUB_MONGODB_HOST', 'localhost:27017')], + host=[os.getenv("MICROBLOGPUB_MONGODB_HOST", "localhost:27017")] ) -DB_NAME = '{}_{}'.format(USERNAME, DOMAIN.replace('.', '_')) +DB_NAME = "{}_{}".format(USERNAME, DOMAIN.replace(".", "_")) DB = mongo_client[DB_NAME] + def _drop_db(): if not DEBUG_MODE: return mongo_client.drop_database(DB_NAME) -KEY = Key(USERNAME, DOMAIN, create=True) + +KEY = get_key(ID, USERNAME, DOMAIN) -JWT_SECRET = get_secret_key('jwt') +JWT_SECRET = get_secret_key("jwt") JWT = JSONWebSignatureSerializer(JWT_SECRET) + def _admin_jwt_token() -> str: - return JWT.dumps({'me': 'ADMIN', 'ts': datetime.now().timestamp()}).decode('utf-8') # type: ignore + return JWT.dumps( # type: ignore + {"me": "ADMIN", "ts": datetime.now().timestamp()} + ).decode( # type: ignore + "utf-8" + ) -ADMIN_API_KEY = get_secret_key('admin_api_key', _admin_jwt_token) +ADMIN_API_KEY = get_secret_key("admin_api_key", _admin_jwt_token) ME = { - "@context": [ - CTX_AS, - CTX_SECURITY, - ], + "@context": [CTX_AS, CTX_SECURITY], "type": "Person", "id": ID, - "following": ID+"/following", - "followers": ID+"/followers", - "liked": ID+"/liked", - "inbox": ID+"/inbox", - "outbox": ID+"/outbox", + "following": ID + "/following", + "followers": ID + "/followers", + "liked": ID + "/liked", + "inbox": ID + "/inbox", + "outbox": ID + "/outbox", "preferredUsername": USERNAME, "name": NAME, "summary": SUMMARY, "endpoints": {}, "url": ID, - "icon": { - "mediaType": "image/png", - "type": "Image", - "url": ICON_URL, - }, - "publicKey": { - "id": ID+"#main-key", - "owner": ID, - "publicKeyPem": KEY.pubkey_pem, - }, + "icon": {"mediaType": "image/png", "type": "Image", "url": ICON_URL}, + "publicKey": KEY.to_dict(), } -print(ME) - -ACTOR_SERVICE = ActorService(USER_AGENT, DB.actors_cache, ID, ME, DB.instances) -OBJECT_SERVICE = ObjectService(USER_AGENT, DB.objects_cache, DB.inbox, DB.outbox, DB.instances) diff --git a/dev-requirements.txt b/dev-requirements.txt index 62e71f2..7db7fab 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,8 @@ +git+https://github.com/tsileo/little-boxes.git pytest requests html2text pyyaml flake8 mypy +black diff --git a/requirements.txt b/requirements.txt index 425405f..eb16141 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,21 +2,19 @@ libsass gunicorn piexif requests -markdown python-u2flib-server Flask Flask-WTF Celery pymongo -pyld timeago bleach -pycryptodome html2text feedgen itsdangerous bcrypt mf2py passlib -pyyaml git+https://github.com/erikriver/opengraph.git +git+https://github.com/tsileo/little-boxes.git +pyyaml diff --git a/tasks.py b/tasks.py index a30854a..a5c85db 100644 --- a/tasks.py +++ b/tasks.py @@ -1,47 +1,52 @@ -import os import json import logging +import os import random import requests from celery import Celery from requests.exceptions import HTTPError -from config import HEADERS -from config import ID from config import DB +from config import HEADERS from config import KEY from config import USER_AGENT -from utils.httpsig import HTTPSigAuth +from little_boxes.httpsig import HTTPSigAuth +from little_boxes.linked_data_sig import generate_signature from utils.opengraph import fetch_og_metadata -from utils.linked_data_sig import generate_signature - log = logging.getLogger(__name__) -app = Celery('tasks', broker=os.getenv('MICROBLOGPUB_AMQP_BROKER', 'pyamqp://guest@localhost//')) -SigAuth = HTTPSigAuth(ID+'#main-key', KEY.privkey) +app = Celery( + "tasks", broker=os.getenv("MICROBLOGPUB_AMQP_BROKER", "pyamqp://guest@localhost//") +) +SigAuth = HTTPSigAuth(KEY) @app.task(bind=True, max_retries=12) def post_to_inbox(self, payload: str, to: str) -> None: try: - log.info('payload=%s', payload) - log.info('generating sig') + log.info("payload=%s", payload) + log.info("generating sig") signed_payload = json.loads(payload) - generate_signature(signed_payload, KEY.privkey) - log.info('to=%s', to) - resp = requests.post(to, data=json.dumps(signed_payload), auth=SigAuth, headers={ - 'Content-Type': HEADERS[1], - 'Accept': HEADERS[1], - 'User-Agent': USER_AGENT, - }) - log.info('resp=%s', resp) - log.info('resp_body=%s', resp.text) + generate_signature(signed_payload, KEY) + log.info("to=%s", to) + resp = requests.post( + to, + data=json.dumps(signed_payload), + auth=SigAuth, + headers={ + "Content-Type": HEADERS[1], + "Accept": HEADERS[1], + "User-Agent": USER_AGENT, + }, + ) + log.info("resp=%s", resp) + log.info("resp_body=%s", resp.text) resp.raise_for_status() except HTTPError as err: - log.exception('request failed') + log.exception("request failed") if 400 >= err.response.status_code >= 499: - log.info('client error, no retry') + log.info("client error, no retry") return self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) @@ -49,11 +54,15 @@ def post_to_inbox(self, payload: str, to: str) -> None: @app.task(bind=True, max_retries=12) def fetch_og(self, col, remote_id): try: - log.info('fetch_og_meta remote_id=%s col=%s', remote_id, col) - if col == 'INBOX': - log.info('%d links saved', fetch_og_metadata(USER_AGENT, DB.inbox, remote_id)) - elif col == 'OUTBOX': - log.info('%d links saved', fetch_og_metadata(USER_AGENT, DB.outbox, remote_id)) + log.info("fetch_og_meta remote_id=%s col=%s", remote_id, col) + if col == "INBOX": + log.info( + "%d links saved", fetch_og_metadata(USER_AGENT, DB.inbox, remote_id) + ) + elif col == "OUTBOX": + log.info( + "%d links saved", fetch_og_metadata(USER_AGENT, DB.outbox, remote_id) + ) except Exception as err: - self.log.exception('failed') + self.log.exception("failed") self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) diff --git a/tests/federation_test.py b/tests/federation_test.py index e050afc..6e0a7ea 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -1,12 +1,12 @@ -import time import os +import time +from typing import List +from typing import Tuple import requests from html2text import html2text -from utils import activitypub_utils -from typing import Tuple -from typing import List +from little_boxes.collection import parse_collection def resp2plaintext(resp): @@ -22,33 +22,38 @@ class Instance(object): self.docker_url = docker_url or host_url self._create_delay = 10 with open( - os.path.join(os.path.dirname(os.path.abspath(__file__)), f'fixtures/{name}/config/admin_api_key.key') + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + f"fixtures/{name}/config/admin_api_key.key", + ) ) as f: api_key = f.read() - self._auth_headers = {'Authorization': f'Bearer {api_key}'} + self._auth_headers = {"Authorization": f"Bearer {api_key}"} - def _do_req(self, url, headers): + def _do_req(self, url): """Used to parse collection.""" url = url.replace(self.docker_url, self.host_url) - resp = requests.get(url, headers=headers) + resp = requests.get(url, headers={'Accept': 'application/activity+json'}) resp.raise_for_status() return resp.json() def _parse_collection(self, payload=None, url=None): """Parses a collection (go through all the pages).""" - return activitypub_utils.parse_collection(url=url, payload=payload, do_req=self._do_req) + return parse_collection( + url=url, payload=payload, fetcher=self._do_req, + ) def ping(self): """Ensures the homepage is reachable.""" - resp = requests.get(f'{self.host_url}/') + resp = requests.get(f"{self.host_url}/") resp.raise_for_status() assert resp.status_code == 200 def debug(self): """Returns the debug infos (number of items in the inbox/outbox.""" resp = requests.get( - f'{self.host_url}/api/debug', - headers={**self._auth_headers, 'Accept': 'application/json'}, + f"{self.host_url}/api/debug", + headers={**self._auth_headers, "Accept": "application/json"}, ) resp.raise_for_status() @@ -57,8 +62,8 @@ class Instance(object): def drop_db(self): """Drops the MongoDB DB.""" resp = requests.delete( - f'{self.host_url}/api/debug', - headers={**self._auth_headers, 'Accept': 'application/json'}, + f"{self.host_url}/api/debug", + headers={**self._auth_headers, "Accept": "application/json"}, ) resp.raise_for_status() @@ -68,100 +73,92 @@ class Instance(object): """Blocks an actor.""" # Instance1 follows instance2 resp = requests.post( - f'{self.host_url}/api/block', - params={'actor': actor_url}, + f"{self.host_url}/api/block", + params={"actor": actor_url}, headers=self._auth_headers, ) assert resp.status_code == 201 # We need to wait for the Follow/Accept dance - time.sleep(self._create_delay/2) - return resp.json().get('activity') + time.sleep(self._create_delay / 2) + return resp.json().get("activity") - def follow(self, instance: 'Instance') -> str: + def follow(self, instance: "Instance") -> str: """Follows another instance.""" # Instance1 follows instance2 resp = requests.post( - f'{self.host_url}/api/follow', - json={'actor': instance.docker_url}, + f"{self.host_url}/api/follow", + json={"actor": instance.docker_url}, headers=self._auth_headers, ) assert resp.status_code == 201 # We need to wait for the Follow/Accept dance time.sleep(self._create_delay) - return resp.json().get('activity') + return resp.json().get("activity") def new_note(self, content, reply=None) -> str: """Creates a new note.""" - params = {'content': content} + params = {"content": content} if reply: - params['reply'] = reply + params["reply"] = reply resp = requests.post( - f'{self.host_url}/api/new_note', - json=params, - headers=self._auth_headers, + f"{self.host_url}/api/new_note", json=params, headers=self._auth_headers ) assert resp.status_code == 201 time.sleep(self._create_delay) - return resp.json().get('activity') + return resp.json().get("activity") def boost(self, oid: str) -> str: """Creates an Announce activity.""" resp = requests.post( - f'{self.host_url}/api/boost', - json={'id': oid}, - headers=self._auth_headers, + f"{self.host_url}/api/boost", json={"id": oid}, headers=self._auth_headers ) assert resp.status_code == 201 time.sleep(self._create_delay) - return resp.json().get('activity') + return resp.json().get("activity") def like(self, oid: str) -> str: """Creates a Like activity.""" resp = requests.post( - f'{self.host_url}/api/like', - json={'id': oid}, - headers=self._auth_headers, + f"{self.host_url}/api/like", json={"id": oid}, headers=self._auth_headers ) assert resp.status_code == 201 time.sleep(self._create_delay) - return resp.json().get('activity') + return resp.json().get("activity") def delete(self, oid: str) -> str: """Creates a Delete activity.""" resp = requests.post( - f'{self.host_url}/api/note/delete', - json={'id': oid}, + f"{self.host_url}/api/note/delete", + json={"id": oid}, headers=self._auth_headers, ) assert resp.status_code == 201 time.sleep(self._create_delay) - return resp.json().get('activity') + return resp.json().get("activity") def undo(self, oid: str) -> str: """Creates a Undo activity.""" resp = requests.post( - f'{self.host_url}/api/undo', - json={'id': oid}, - headers=self._auth_headers, + f"{self.host_url}/api/undo", json={"id": oid}, headers=self._auth_headers ) assert resp.status_code == 201 # We need to wait for the Follow/Accept dance time.sleep(self._create_delay) - return resp.json().get('activity') + return resp.json().get("activity") def followers(self) -> List[str]: """Parses the followers collection.""" resp = requests.get( - f'{self.host_url}/followers', - headers={'Accept': 'application/activity+json'}, + f"{self.host_url}/followers", + headers={"Accept": "application/activity+json"}, ) resp.raise_for_status() @@ -172,8 +169,8 @@ class Instance(object): def following(self): """Parses the following collection.""" resp = requests.get( - f'{self.host_url}/following', - headers={'Accept': 'application/activity+json'}, + f"{self.host_url}/following", + headers={"Accept": "application/activity+json"}, ) resp.raise_for_status() @@ -184,8 +181,8 @@ class Instance(object): def outbox(self): """Returns the instance outbox.""" resp = requests.get( - f'{self.host_url}/following', - headers={'Accept': 'application/activity+json'}, + f"{self.host_url}/following", + headers={"Accept": "application/activity+json"}, ) resp.raise_for_status() return resp.json() @@ -194,7 +191,7 @@ class Instance(object): """Fetches a specific item from the instance outbox.""" resp = requests.get( aid.replace(self.docker_url, self.host_url), - headers={'Accept': 'application/activity+json'}, + headers={"Accept": "application/activity+json"}, ) resp.raise_for_status() return resp.json() @@ -202,8 +199,8 @@ class Instance(object): def stream_jsonfeed(self): """Returns the "stream"'s JSON feed.""" resp = requests.get( - f'{self.host_url}/api/stream', - headers={**self._auth_headers, 'Accept': 'application/json'}, + f"{self.host_url}/api/stream", + headers={**self._auth_headers, "Accept": "application/json"}, ) resp.raise_for_status() return resp.json() @@ -211,10 +208,14 @@ class Instance(object): def _instances() -> Tuple[Instance, Instance]: """Initializes the client for the two test instances.""" - instance1 = Instance('instance1', 'http://localhost:5006', 'http://instance1_web_1:5005') + instance1 = Instance( + "instance1", "http://localhost:5006", "http://instance1_web_1:5005" + ) instance1.ping() - instance2 = Instance('instance2', 'http://localhost:5007', 'http://instance2_web_1:5005') + instance2 = Instance( + "instance2", "http://localhost:5007", "http://instance2_web_1:5005" + ) instance2.ping() # Return the DB @@ -230,12 +231,12 @@ def test_follow() -> None: # Instance1 follows instance2 instance1.follow(instance2) instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 1 # An Accept activity should be there - assert instance1_debug['outbox'] == 1 # We've sent a Follow activity + assert instance1_debug["inbox"] == 1 # An Accept activity should be there + assert instance1_debug["outbox"] == 1 # We've sent a Follow activity instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 1 # An Follow activity should be there - assert instance2_debug['outbox'] == 1 # We've sent a Accept activity + assert instance2_debug["inbox"] == 1 # An Follow activity should be there + assert instance2_debug["outbox"] == 1 # We've sent a Accept activity assert instance2.followers() == [instance1.docker_url] assert instance1.following() == [instance2.docker_url] @@ -247,12 +248,12 @@ def test_follow_unfollow(): # Instance1 follows instance2 follow_id = instance1.follow(instance2) instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 1 # An Accept activity should be there - assert instance1_debug['outbox'] == 1 # We've sent a Follow activity + assert instance1_debug["inbox"] == 1 # An Accept activity should be there + assert instance1_debug["outbox"] == 1 # We've sent a Follow activity instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 1 # An Follow activity should be there - assert instance2_debug['outbox'] == 1 # We've sent a Accept activity + assert instance2_debug["inbox"] == 1 # An Follow activity should be there + assert instance2_debug["outbox"] == 1 # We've sent a Accept activity assert instance2.followers() == [instance1.docker_url] assert instance1.following() == [instance2.docker_url] @@ -263,12 +264,12 @@ def test_follow_unfollow(): assert instance1.following() == [] instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 1 # An Accept activity should be there - assert instance1_debug['outbox'] == 2 # We've sent a Follow and a Undo activity + assert instance1_debug["inbox"] == 1 # An Accept activity should be there + assert instance1_debug["outbox"] == 2 # We've sent a Follow and a Undo activity instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 2 # An Follow and Undo activity should be there - assert instance2_debug['outbox'] == 1 # We've sent a Accept activity + assert instance2_debug["inbox"] == 2 # An Follow and Undo activity should be there + assert instance2_debug["outbox"] == 1 # We've sent a Accept activity def test_post_content(): @@ -279,17 +280,19 @@ def test_post_content(): instance2.follow(instance1) inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 - create_id = instance1.new_note('hello') + create_id = instance1.new_note("hello") instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + assert ( + instance2_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert instance2_debug["outbox"] == 2 # We've sent a Accept and a Follow activity # Ensure the post is visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 1 - assert inbox_stream['items'][0]['id'] == create_id + assert len(inbox_stream["items"]) == 1 + assert inbox_stream["items"][0]["id"] == create_id def test_block_and_post_content(): @@ -300,18 +303,22 @@ def test_block_and_post_content(): instance2.follow(instance1) inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 instance2.block(instance1.docker_url) - instance1.new_note('hello') + instance1.new_note("hello") instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 2 # An Follow, Accept activity should be there, Create should have been dropped - assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow activity + the Block activity + assert ( + instance2_debug["inbox"] == 2 + ) # An Follow, Accept activity should be there, Create should have been dropped + assert ( + instance2_debug["outbox"] == 3 + ) # We've sent a Accept and a Follow activity + the Block activity # Ensure the post is not visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 def test_post_content_and_delete(): @@ -322,26 +329,30 @@ def test_post_content_and_delete(): instance2.follow(instance1) inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 - create_id = instance1.new_note('hello') + create_id = instance1.new_note("hello") instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + assert ( + instance2_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert instance2_debug["outbox"] == 2 # We've sent a Accept and a Follow activity # Ensure the post is visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 1 - assert inbox_stream['items'][0]['id'] == create_id + assert len(inbox_stream["items"]) == 1 + assert inbox_stream["items"][0]["id"] == create_id - instance1.delete(f'{create_id}/activity') + instance1.delete(f"{create_id}/activity") instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 4 # An Follow, Accept and Create and Delete activity should be there - assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + assert ( + instance2_debug["inbox"] == 4 + ) # An Follow, Accept and Create and Delete activity should be there + assert instance2_debug["outbox"] == 2 # We've sent a Accept and a Follow activity # Ensure the post has been delete from instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 def test_post_content_and_like(): @@ -351,26 +362,26 @@ def test_post_content_and_like(): instance1.follow(instance2) instance2.follow(instance1) - create_id = instance1.new_note('hello') + create_id = instance1.new_note("hello") # Ensure the post is visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 1 - assert inbox_stream['items'][0]['id'] == create_id + assert len(inbox_stream["items"]) == 1 + assert inbox_stream["items"][0]["id"] == create_id # Now, instance2 like the note - like_id = instance2.like(f'{create_id}/activity') + like_id = instance2.like(f"{create_id}/activity") instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 3 # Follow, Accept and Like - assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + assert instance1_debug["inbox"] == 3 # Follow, Accept and Like + assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create - note = instance1.outbox_get(f'{create_id}/activity') - assert 'likes' in note - assert note['likes']['totalItems'] == 1 - likes = instance1._parse_collection(url=note['likes']['first']) + note = instance1.outbox_get(f"{create_id}/activity") + assert "likes" in note + assert note["likes"]["totalItems"] == 1 + likes = instance1._parse_collection(url=note["likes"]["first"]) assert len(likes) == 1 - assert likes[0]['id'] == like_id + assert likes[0]["id"] == like_id def test_post_content_and_like_unlike() -> None: @@ -380,36 +391,36 @@ def test_post_content_and_like_unlike() -> None: instance1.follow(instance2) instance2.follow(instance1) - create_id = instance1.new_note('hello') + create_id = instance1.new_note("hello") # Ensure the post is visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 1 - assert inbox_stream['items'][0]['id'] == create_id + assert len(inbox_stream["items"]) == 1 + assert inbox_stream["items"][0]["id"] == create_id # Now, instance2 like the note - like_id = instance2.like(f'{create_id}/activity') + like_id = instance2.like(f"{create_id}/activity") instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 3 # Follow, Accept and Like - assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + assert instance1_debug["inbox"] == 3 # Follow, Accept and Like + assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create - note = instance1.outbox_get(f'{create_id}/activity') - assert 'likes' in note - assert note['likes']['totalItems'] == 1 - likes = instance1._parse_collection(url=note['likes']['first']) + note = instance1.outbox_get(f"{create_id}/activity") + assert "likes" in note + assert note["likes"]["totalItems"] == 1 + likes = instance1._parse_collection(url=note["likes"]["first"]) assert len(likes) == 1 - assert likes[0]['id'] == like_id + assert likes[0]["id"] == like_id instance2.undo(like_id) instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 4 # Follow, Accept and Like and Undo - assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + assert instance1_debug["inbox"] == 4 # Follow, Accept and Like and Undo + assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create - note = instance1.outbox_get(f'{create_id}/activity') - assert 'likes' in note - assert note['likes']['totalItems'] == 0 + note = instance1.outbox_get(f"{create_id}/activity") + assert "likes" in note + assert note["likes"]["totalItems"] == 0 def test_post_content_and_boost() -> None: @@ -419,26 +430,26 @@ def test_post_content_and_boost() -> None: instance1.follow(instance2) instance2.follow(instance1) - create_id = instance1.new_note('hello') + create_id = instance1.new_note("hello") # Ensure the post is visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 1 - assert inbox_stream['items'][0]['id'] == create_id + assert len(inbox_stream["items"]) == 1 + assert inbox_stream["items"][0]["id"] == create_id # Now, instance2 like the note - boost_id = instance2.boost(f'{create_id}/activity') + boost_id = instance2.boost(f"{create_id}/activity") instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 3 # Follow, Accept and Announce - assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + assert instance1_debug["inbox"] == 3 # Follow, Accept and Announce + assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create - note = instance1.outbox_get(f'{create_id}/activity') - assert 'shares' in note - assert note['shares']['totalItems'] == 1 - shares = instance1._parse_collection(url=note['shares']['first']) + note = instance1.outbox_get(f"{create_id}/activity") + assert "shares" in note + assert note["shares"]["totalItems"] == 1 + shares = instance1._parse_collection(url=note["shares"]["first"]) assert len(shares) == 1 - assert shares[0]['id'] == boost_id + assert shares[0]["id"] == boost_id def test_post_content_and_boost_unboost() -> None: @@ -448,36 +459,36 @@ def test_post_content_and_boost_unboost() -> None: instance1.follow(instance2) instance2.follow(instance1) - create_id = instance1.new_note('hello') + create_id = instance1.new_note("hello") # Ensure the post is visible in instance2's stream inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 1 - assert inbox_stream['items'][0]['id'] == create_id + assert len(inbox_stream["items"]) == 1 + assert inbox_stream["items"][0]["id"] == create_id # Now, instance2 like the note - boost_id = instance2.boost(f'{create_id}/activity') + boost_id = instance2.boost(f"{create_id}/activity") instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 3 # Follow, Accept and Announce - assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + assert instance1_debug["inbox"] == 3 # Follow, Accept and Announce + assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create - note = instance1.outbox_get(f'{create_id}/activity') - assert 'shares' in note - assert note['shares']['totalItems'] == 1 - shares = instance1._parse_collection(url=note['shares']['first']) + note = instance1.outbox_get(f"{create_id}/activity") + assert "shares" in note + assert note["shares"]["totalItems"] == 1 + shares = instance1._parse_collection(url=note["shares"]["first"]) assert len(shares) == 1 - assert shares[0]['id'] == boost_id + assert shares[0]["id"] == boost_id instance2.undo(boost_id) instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 4 # Follow, Accept and Announce and Undo - assert instance1_debug['outbox'] == 3 # Folllow, Accept, and Create + assert instance1_debug["inbox"] == 4 # Follow, Accept and Announce and Undo + assert instance1_debug["outbox"] == 3 # Folllow, Accept, and Create - note = instance1.outbox_get(f'{create_id}/activity') - assert 'shares' in note - assert note['shares']['totalItems'] == 0 + note = instance1.outbox_get(f"{create_id}/activity") + assert "shares" in note + assert note["shares"]["totalItems"] == 0 def test_post_content_and_post_reply() -> None: @@ -488,40 +499,50 @@ def test_post_content_and_post_reply() -> None: instance2.follow(instance1) inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 - instance1_create_id = instance1.new_note('hello') + instance1_create_id = instance1.new_note("hello") instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + assert ( + instance2_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert instance2_debug["outbox"] == 2 # We've sent a Accept and a Follow activity # Ensure the post is visible in instance2's stream instance2_inbox_stream = instance2.stream_jsonfeed() - assert len(instance2_inbox_stream['items']) == 1 - assert instance2_inbox_stream['items'][0]['id'] == instance1_create_id + assert len(instance2_inbox_stream["items"]) == 1 + assert instance2_inbox_stream["items"][0]["id"] == instance1_create_id instance2_create_id = instance2.new_note( - f'hey @instance1@{instance1.docker_url}', - reply=f'{instance1_create_id}/activity', + f"hey @instance1@{instance1.docker_url}", + reply=f"{instance1_create_id}/activity", ) instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity + assert ( + instance2_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert ( + instance2_debug["outbox"] == 3 + ) # We've sent a Accept and a Follow and a Create activity instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance1_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity + assert ( + instance1_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert ( + instance1_debug["outbox"] == 3 + ) # We've sent a Accept and a Follow and a Create activity instance1_inbox_stream = instance1.stream_jsonfeed() - assert len(instance1_inbox_stream['items']) == 1 - assert instance1_inbox_stream['items'][0]['id'] == instance2_create_id + assert len(instance1_inbox_stream["items"]) == 1 + assert instance1_inbox_stream["items"][0]["id"] == instance2_create_id - instance1_note = instance1.outbox_get(f'{instance1_create_id}/activity') - assert 'replies' in instance1_note - assert instance1_note['replies']['totalItems'] == 1 - replies = instance1._parse_collection(url=instance1_note['replies']['first']) + instance1_note = instance1.outbox_get(f"{instance1_create_id}/activity") + assert "replies" in instance1_note + assert instance1_note["replies"]["totalItems"] == 1 + replies = instance1._parse_collection(url=instance1_note["replies"]["first"]) assert len(replies) == 1 - assert replies[0]['id'] == f'{instance2_create_id}/activity' + assert replies[0]["id"] == f"{instance2_create_id}/activity" def test_post_content_and_post_reply_and_delete() -> None: @@ -532,44 +553,58 @@ def test_post_content_and_post_reply_and_delete() -> None: instance2.follow(instance1) inbox_stream = instance2.stream_jsonfeed() - assert len(inbox_stream['items']) == 0 + assert len(inbox_stream["items"]) == 0 - instance1_create_id = instance1.new_note('hello') + instance1_create_id = instance1.new_note("hello") instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance2_debug['outbox'] == 2 # We've sent a Accept and a Follow activity + assert ( + instance2_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert instance2_debug["outbox"] == 2 # We've sent a Accept and a Follow activity # Ensure the post is visible in instance2's stream instance2_inbox_stream = instance2.stream_jsonfeed() - assert len(instance2_inbox_stream['items']) == 1 - assert instance2_inbox_stream['items'][0]['id'] == instance1_create_id + assert len(instance2_inbox_stream["items"]) == 1 + assert instance2_inbox_stream["items"][0]["id"] == instance1_create_id instance2_create_id = instance2.new_note( - f'hey @instance1@{instance1.docker_url}', - reply=f'{instance1_create_id}/activity', + f"hey @instance1@{instance1.docker_url}", + reply=f"{instance1_create_id}/activity", ) instance2_debug = instance2.debug() - assert instance2_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance2_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity + assert ( + instance2_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert ( + instance2_debug["outbox"] == 3 + ) # We've sent a Accept and a Follow and a Create activity instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 3 # An Follow, Accept and Create activity should be there - assert instance1_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity + assert ( + instance1_debug["inbox"] == 3 + ) # An Follow, Accept and Create activity should be there + assert ( + instance1_debug["outbox"] == 3 + ) # We've sent a Accept and a Follow and a Create activity instance1_inbox_stream = instance1.stream_jsonfeed() - assert len(instance1_inbox_stream['items']) == 1 - assert instance1_inbox_stream['items'][0]['id'] == instance2_create_id + assert len(instance1_inbox_stream["items"]) == 1 + assert instance1_inbox_stream["items"][0]["id"] == instance2_create_id - instance1_note = instance1.outbox_get(f'{instance1_create_id}/activity') - assert 'replies' in instance1_note - assert instance1_note['replies']['totalItems'] == 1 + instance1_note = instance1.outbox_get(f"{instance1_create_id}/activity") + assert "replies" in instance1_note + assert instance1_note["replies"]["totalItems"] == 1 - instance2.delete(f'{instance2_create_id}/activity') + instance2.delete(f"{instance2_create_id}/activity") instance1_debug = instance1.debug() - assert instance1_debug['inbox'] == 4 # An Follow, Accept and Create and Delete activity should be there - assert instance1_debug['outbox'] == 3 # We've sent a Accept and a Follow and a Create activity + assert ( + instance1_debug["inbox"] == 4 + ) # An Follow, Accept and Create and Delete activity should be there + assert ( + instance1_debug["outbox"] == 3 + ) # We've sent a Accept and a Follow and a Create activity - instance1_note = instance1.outbox_get(f'{instance1_create_id}/activity') - assert 'replies' in instance1_note - assert instance1_note['replies']['totalItems'] == 0 + instance1_note = instance1.outbox_get(f"{instance1_create_id}/activity") + assert "replies" in instance1_note + assert instance1_note["replies"]["totalItems"] == 0 diff --git a/tests/integration_test.py b/tests/integration_test.py index 4270b4b..dbfe19b 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -9,7 +9,10 @@ from html2text import html2text def config(): """Return the current config as a dict.""" import yaml - with open(os.path.join(os.path.dirname(__file__), '..', 'config/me.yml'), 'rb') as f: + + with open( + os.path.join(os.path.dirname(__file__), "..", "config/me.yml"), "rb" + ) as f: yield yaml.load(f) @@ -20,9 +23,9 @@ def resp2plaintext(resp): def test_ping_homepage(config): """Ensure the homepage is accessible.""" - resp = requests.get('http://localhost:5005') + resp = requests.get("http://localhost:5005") resp.raise_for_status() assert resp.status_code == 200 body = resp2plaintext(resp) - assert config['name'] in body + assert config["name"] in body assert f"@{config['username']}@{config['domain']}" in body diff --git a/utils/__init__.py b/utils/__init__.py index c30c37d..cdf368d 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -4,9 +4,9 @@ logger = logging.getLogger(__name__) def strtobool(s: str) -> bool: - if s in ['y', 'yes', 'true', 'on', '1']: + if s in ["y", "yes", "true", "on", "1"]: return True - if s in ['n', 'no', 'false', 'off', '0']: + if s in ["n", "no", "false", "off", "0"]: return False - raise ValueError(f'cannot convert {s} to bool') + raise ValueError(f"cannot convert {s} to bool") diff --git a/utils/activitypub_utils.py b/utils/activitypub_utils.py deleted file mode 100644 index 0275f54..0000000 --- a/utils/activitypub_utils.py +++ /dev/null @@ -1,65 +0,0 @@ -from typing import Optional, Dict, List, Any - -import requests - -from .errors import RecursionLimitExceededError -from .errors import UnexpectedActivityTypeError - - -def _do_req(url: str, headers: Dict[str, str]) -> Dict[str, Any]: - resp = requests.get(url, headers=headers) - resp.raise_for_status() - return resp.json() - - -def parse_collection( - payload: Optional[Dict[str, Any]] = None, - url: Optional[str] = None, - user_agent: Optional[str] = None, - level: int = 0, - do_req: Any = _do_req, -) -> List[str]: - """Resolve/fetch a `Collection`/`OrderedCollection`.""" - if level > 3: - raise RecursionLimitExceededError('recursion limit exceeded') - - # Go through all the pages - headers = {'Accept': 'application/activity+json'} - if user_agent: - headers['User-Agent'] = user_agent - - out: List[str] = [] - if url: - payload = do_req(url, headers) - if not payload: - raise ValueError('must at least prove a payload or an URL') - - if payload['type'] in ['Collection', 'OrderedCollection']: - if 'orderedItems' in payload: - return payload['orderedItems'] - if 'items' in payload: - return payload['items'] - if 'first' in payload: - if 'orderedItems' in payload['first']: - out.extend(payload['first']['orderedItems']) - if 'items' in payload['first']: - out.extend(payload['first']['items']) - n = payload['first'].get('next') - if n: - out.extend(parse_collection(url=n, user_agent=user_agent, level=level+1, do_req=do_req)) - return out - - while payload: - if payload['type'] in ['CollectionPage', 'OrderedCollectionPage']: - if 'orderedItems' in payload: - out.extend(payload['orderedItems']) - if 'items' in payload: - out.extend(payload['items']) - n = payload.get('next') - if n is None: - break - payload = do_req(n, headers) - else: - raise UnexpectedActivityTypeError('unexpected activity type {}'.format(payload['type'])) - - return out diff --git a/utils/actor_service.py b/utils/actor_service.py deleted file mode 100644 index 9982235..0000000 --- a/utils/actor_service.py +++ /dev/null @@ -1,81 +0,0 @@ -import logging - -import requests -from urllib.parse import urlparse -from Crypto.PublicKey import RSA - -from .urlutils import check_url -from .errors import ActivityNotFoundError - -logger = logging.getLogger(__name__) - - -class NotAnActorError(Exception): - def __init__(self, activity): - self.activity = activity - - -class ActorService(object): - def __init__(self, user_agent, col, actor_id, actor_data, instances): - logger.debug(f'Initializing ActorService user_agent={user_agent}') - self._user_agent = user_agent - self._col = col - self._in_mem = {actor_id: actor_data} - self._instances = instances - self._known_instances = set() - - def _fetch(self, actor_url): - logger.debug(f'fetching remote object {actor_url}') - - check_url(actor_url) - - resp = requests.get(actor_url, headers={ - 'Accept': 'application/activity+json', - 'User-Agent': self._user_agent, - }) - if resp.status_code == 404: - raise ActivityNotFoundError(f'{actor_url} cannot be fetched, 404 not found error') - - resp.raise_for_status() - return resp.json() - - def get(self, actor_url, reload_cache=False): - logger.info(f'get actor {actor_url} (reload_cache={reload_cache})') - - if actor_url in self._in_mem: - return self._in_mem[actor_url] - - instance = urlparse(actor_url)._replace(path='', query='', fragment='').geturl() - if instance not in self._known_instances: - self._known_instances.add(instance) - if not self._instances.find_one({'instance': instance}): - self._instances.insert({'instance': instance, 'first_object': actor_url}) - - if reload_cache: - actor = self._fetch(actor_url) - self._in_mem[actor_url] = actor - self._col.update({'actor_id': actor_url}, {'$set': {'cached_response': actor}}, upsert=True) - return actor - - cached_actor = self._col.find_one({'actor_id': actor_url}) - if cached_actor: - return cached_actor['cached_response'] - - actor = self._fetch(actor_url) - if not 'type' in actor: - raise NotAnActorError(None) - if actor['type'] != 'Person': - raise NotAnActorError(actor) - - self._col.update({'actor_id': actor_url}, {'$set': {'cached_response': actor}}, upsert=True) - self._in_mem[actor_url] = actor - return actor - - def get_public_key(self, actor_url, reload_cache=False): - profile = self.get(actor_url, reload_cache=reload_cache) - pub = profile['publicKey'] - return pub['id'], RSA.importKey(pub['publicKeyPem']) - - def get_inbox_url(self, actor_url, reload_cache=False): - profile = self.get(actor_url, reload_cache=reload_cache) - return profile.get('inbox') diff --git a/utils/content_helper.py b/utils/content_helper.py deleted file mode 100644 index b254e2b..0000000 --- a/utils/content_helper.py +++ /dev/null @@ -1,58 +0,0 @@ -import typing -import re - -from bleach.linkifier import Linker -from markdown import markdown - -from utils.webfinger import get_actor_url -from config import USERNAME, BASE_URL, ID -from config import ACTOR_SERVICE - -from typing import List, Optional, Tuple, Dict, Any, Union, Type - - -def set_attrs(attrs, new=False): - attrs[(None, u'target')] = u'_blank' - attrs[(None, u'class')] = u'external' - attrs[(None, u'rel')] = u'noopener' - attrs[(None, u'title')] = attrs[(None, u'href')] - return attrs - - -LINKER = Linker(callbacks=[set_attrs]) -HASHTAG_REGEX = re.compile(r"(#[\d\w\.]+)") -MENTION_REGEX = re.compile(r"@[\d\w_.+-]+@[\d\w-]+\.[\d\w\-.]+") - - -def hashtagify(content: str) -> Tuple[str, List[Dict[str, str]]]: - tags = [] - for hashtag in re.findall(HASHTAG_REGEX, content): - tag = hashtag[1:] - link = f'' - tags.append(dict(href=f'{BASE_URL}/tags/{tag}', name=hashtag, type='Hashtag')) - content = content.replace(hashtag, link) - return content, tags - - -def mentionify(content: str) -> Tuple[str, List[Dict[str, str]]]: - tags = [] - for mention in re.findall(MENTION_REGEX, content): - _, username, domain = mention.split('@') - actor_url = get_actor_url(mention) - p = ACTOR_SERVICE.get(actor_url) - print(p) - tags.append(dict(type='Mention', href=p['id'], name=mention)) - link = f'@{username}' - content = content.replace(mention, link) - return content, tags - - -def parse_markdown(content: str) -> Tuple[str, List[Dict[str, str]]]: - tags = [] - content = LINKER.linkify(content) - content, hashtag_tags = hashtagify(content) - tags.extend(hashtag_tags) - content, mention_tags = mentionify(content) - tags.extend(mention_tags) - content = markdown(content) - return content, tags diff --git a/utils/errors.py b/utils/errors.py deleted file mode 100644 index 7ffe744..0000000 --- a/utils/errors.py +++ /dev/null @@ -1,37 +0,0 @@ - -class Error(Exception): - status_code = 400 - - def __init__(self, message, status_code=None, payload=None): - Exception.__init__(self) - self.message = message - if status_code is not None: - self.status_code = status_code - self.payload = payload - - def to_dict(self): - rv = dict(self.payload or ()) - rv['message'] = self.message - return rv - - def __repr__(self): - return f'{self.__class__.__qualname__}({self.message!r}, payload={self.payload!r}, status_code={self.status_code})' - - -class NotFromOutboxError(Error): - pass - -class ActivityNotFoundError(Error): - status_code = 404 - - -class BadActivityError(Error): - pass - - -class RecursionLimitExceededError(BadActivityError): - pass - - -class UnexpectedActivityTypeError(BadActivityError): - pass diff --git a/utils/httpsig.py b/utils/httpsig.py deleted file mode 100644 index 8437784..0000000 --- a/utils/httpsig.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Implements HTTP signature for Flask requests. - -Mastodon instances won't accept requests that are not signed using this scheme. - -""" -from datetime import datetime -from urllib.parse import urlparse -from typing import Any, Dict, Optional -import base64 -import hashlib -import logging - -from flask import request -from requests.auth import AuthBase - -from Crypto.Signature import PKCS1_v1_5 -from Crypto.Hash import SHA256 - -logger = logging.getLogger(__name__) - - -def _build_signed_string(signed_headers: str, method: str, path: str, headers: Any, body_digest: str) -> str: - out = [] - for signed_header in signed_headers.split(' '): - if signed_header == '(request-target)': - out.append('(request-target): '+method.lower()+' '+path) - elif signed_header == 'digest': - out.append('digest: '+body_digest) - else: - out.append(signed_header+': '+headers[signed_header]) - return '\n'.join(out) - - -def _parse_sig_header(val: Optional[str]) -> Optional[Dict[str, str]]: - if not val: - return None - out = {} - for data in val.split(','): - k, v = data.split('=', 1) - out[k] = v[1:len(v)-1] - return out - - -def _verify_h(signed_string, signature, pubkey): - signer = PKCS1_v1_5.new(pubkey) - digest = SHA256.new() - digest.update(signed_string.encode('utf-8')) - return signer.verify(digest, signature) - - -def _body_digest() -> str: - h = hashlib.new('sha256') - h.update(request.data) - return 'SHA-256='+base64.b64encode(h.digest()).decode('utf-8') - - -def verify_request(actor_service) -> bool: - hsig = _parse_sig_header(request.headers.get('Signature')) - if not hsig: - logger.debug('no signature in header') - return False - logger.debug(f'hsig={hsig}') - signed_string = _build_signed_string(hsig['headers'], request.method, request.path, request.headers, _body_digest()) - _, rk = actor_service.get_public_key(hsig['keyId']) - return _verify_h(signed_string, base64.b64decode(hsig['signature']), rk) - - -class HTTPSigAuth(AuthBase): - def __init__(self, keyid, privkey): - self.keyid = keyid - self.privkey = privkey - - def __call__(self, r): - logger.info(f'keyid={self.keyid}') - host = urlparse(r.url).netloc - bh = hashlib.new('sha256') - bh.update(r.body.encode('utf-8')) - bodydigest = 'SHA-256='+base64.b64encode(bh.digest()).decode('utf-8') - date = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT') - r.headers.update({'Digest': bodydigest, 'Date': date}) - r.headers.update({'Host': host}) - sigheaders = '(request-target) user-agent host date digest content-type' - to_be_signed = _build_signed_string(sigheaders, r.method, r.path_url, r.headers, bodydigest) - signer = PKCS1_v1_5.new(self.privkey) - digest = SHA256.new() - digest.update(to_be_signed.encode('utf-8')) - sig = base64.b64encode(signer.sign(digest)) - sig = sig.decode('utf-8') - headers = { - 'Signature': f'keyId="{self.keyid}",algorithm="rsa-sha256",headers="{sigheaders}",signature="{sig}"' - } - logger.info(f'signed request headers={headers}') - r.headers.update(headers) - return r diff --git a/utils/key.py b/utils/key.py index f5a2455..e7012ae 100644 --- a/utils/key.py +++ b/utils/key.py @@ -1,22 +1,22 @@ -import os import binascii - -from Crypto.PublicKey import RSA +import os from typing import Callable -KEY_DIR = os.path.join( - os.path.dirname(os.path.abspath(__file__)), '..', 'config' -) +from little_boxes.key import Key + +KEY_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "config") def _new_key() -> str: - return binascii.hexlify(os.urandom(32)).decode('utf-8') + return binascii.hexlify(os.urandom(32)).decode("utf-8") + def get_secret_key(name: str, new_key: Callable[[], str] = _new_key) -> str: - key_path = os.path.join(KEY_DIR, f'{name}.key') + """Loads or generates a cryptographic key.""" + key_path = os.path.join(KEY_DIR, f"{name}.key") if not os.path.exists(key_path): k = new_key() - with open(key_path, 'w+') as f: + with open(key_path, "w+") as f: f.write(k) return k @@ -24,23 +24,19 @@ def get_secret_key(name: str, new_key: Callable[[], str] = _new_key) -> str: return f.read() -class Key(object): - DEFAULT_KEY_SIZE = 2048 - def __init__(self, user: str, domain: str, create: bool = True) -> None: - user = user.replace('.', '_') - domain = domain.replace('.', '_') - key_path = os.path.join(KEY_DIR, f'key_{user}_{domain}.pem') - if os.path.isfile(key_path): - with open(key_path) as f: - self.privkey_pem = f.read() - self.privkey = RSA.importKey(self.privkey_pem) - self.pubkey_pem = self.privkey.publickey().exportKey('PEM').decode('utf-8') - else: - if not create: - raise Exception('must init private key first') - k = RSA.generate(self.DEFAULT_KEY_SIZE) - self.privkey_pem = k.exportKey('PEM').decode('utf-8') - self.pubkey_pem = k.publickey().exportKey('PEM').decode('utf-8') - with open(key_path, 'w') as f: - f.write(self.privkey_pem) - self.privkey = k +def get_key(owner: str, user: str, domain: str) -> Key: + """"Loads or generates an RSA key.""" + k = Key(owner) + user = user.replace(".", "_") + domain = domain.replace(".", "_") + key_path = os.path.join(KEY_DIR, f"key_{user}_{domain}.pem") + if os.path.isfile(key_path): + with open(key_path) as f: + privkey_pem = f.read() + k.load(privkey_pem) + else: + k.new() + with open(key_path, "w") as f: + f.write(k.privkey_pem) + + return k diff --git a/utils/linked_data_sig.py b/utils/linked_data_sig.py deleted file mode 100644 index 834c9bd..0000000 --- a/utils/linked_data_sig.py +++ /dev/null @@ -1,70 +0,0 @@ -from pyld import jsonld -import hashlib -from datetime import datetime - -from Crypto.Signature import PKCS1_v1_5 -from Crypto.Hash import SHA256 -import base64 - -from typing import Any, Dict - - -# cache the downloaded "schemas", otherwise the library is super slow -# (https://github.com/digitalbazaar/pyld/issues/70) -_CACHE: Dict[str, Any] = {} -LOADER = jsonld.requests_document_loader() - -def _caching_document_loader(url: str) -> Any: - if url in _CACHE: - return _CACHE[url] - resp = LOADER(url) - _CACHE[url] = resp - return resp - -jsonld.set_document_loader(_caching_document_loader) - - -def options_hash(doc): - doc = dict(doc['signature']) - for k in ['type', 'id', 'signatureValue']: - if k in doc: - del doc[k] - doc['@context'] = 'https://w3id.org/identity/v1' - normalized = jsonld.normalize(doc, {'algorithm': 'URDNA2015', 'format': 'application/nquads'}) - h = hashlib.new('sha256') - h.update(normalized.encode('utf-8')) - return h.hexdigest() - - -def doc_hash(doc): - doc = dict(doc) - if 'signature' in doc: - del doc['signature'] - normalized = jsonld.normalize(doc, {'algorithm': 'URDNA2015', 'format': 'application/nquads'}) - h = hashlib.new('sha256') - h.update(normalized.encode('utf-8')) - return h.hexdigest() - - -def verify_signature(doc, pubkey): - to_be_signed = options_hash(doc) + doc_hash(doc) - signature = doc['signature']['signatureValue'] - signer = PKCS1_v1_5.new(pubkey) - digest = SHA256.new() - digest.update(to_be_signed.encode('utf-8')) - return signer.verify(digest, base64.b64decode(signature)) - - -def generate_signature(doc, privkey): - options = { - 'type': 'RsaSignature2017', - 'creator': doc['actor'] + '#main-key', - 'created': datetime.utcnow().replace(microsecond=0).isoformat() + 'Z', - } - doc['signature'] = options - to_be_signed = options_hash(doc) + doc_hash(doc) - signer = PKCS1_v1_5.new(privkey) - digest = SHA256.new() - digest.update(to_be_signed.encode('utf-8')) - sig = base64.b64encode(signer.sign(digest)) - options['signatureValue'] = sig.decode('utf-8') diff --git a/utils/object_service.py b/utils/object_service.py index 1ebc0ce..e46f9b1 100644 --- a/utils/object_service.py +++ b/utils/object_service.py @@ -1,67 +1,21 @@ -import requests -from urllib.parse import urlparse +import logging -from .urlutils import check_url -from .errors import ActivityNotFoundError +from little_boxes.activitypub import get_backend + +logger = logging.getLogger(__name__) class ObjectService(object): - def __init__(self, user_agent, col, inbox, outbox, instances): - self._user_agent = user_agent - self._col = col - self._inbox = inbox - self._outbox = outbox - self._instances = instances - self._known_instances = set() + def __init__(self): + logger.debug("Initializing ObjectService") + self._cache = {} - def _fetch_remote(self, object_id): - print(f'fetch remote {object_id}') - check_url(object_id) - resp = requests.get(object_id, headers={ - 'Accept': 'application/activity+json', - 'User-Agent': self._user_agent, - }) - if resp.status_code == 404: - raise ActivityNotFoundError(f'{object_id} cannot be fetched, 404 error not found') + def get(self, iri, reload_cache=False): + logger.info(f"get actor {iri} (reload_cache={reload_cache})") - resp.raise_for_status() - return resp.json() - - def _fetch(self, object_id): - instance = urlparse(object_id)._replace(path='', query='', fragment='').geturl() - if instance not in self._known_instances: - self._known_instances.add(instance) - if not self._instances.find_one({'instance': instance}): - self._instances.insert({'instance': instance, 'first_object': object_id}) - - obj = self._inbox.find_one({'$or': [{'remote_id': object_id}, {'type': 'Create', 'activity.object.id': object_id}]}) - if obj: - if obj['remote_id'] == object_id: - return obj['activity'] - return obj['activity']['object'] - - obj = self._outbox.find_one({'$or': [{'remote_id': object_id}, {'type': 'Create', 'activity.object.id': object_id}]}) - if obj: - if obj['remote_id'] == object_id: - return obj['activity'] - return obj['activity']['object'] - - return self._fetch_remote(object_id) - - def get(self, object_id, reload_cache=False, part_of_stream=False, announce_published=None): - if reload_cache: - obj = self._fetch(object_id) - self._col.update({'object_id': object_id}, {'$set': {'cached_object': obj, 'meta.part_of_stream': part_of_stream, 'meta.announce_published': announce_published}}, upsert=True) - return obj - - cached_object = self._col.find_one({'object_id': object_id}) - if cached_object: - print(f'ObjectService: {cached_object}') - return cached_object['cached_object'] - - obj = self._fetch(object_id) - - self._col.update({'object_id': object_id}, {'$set': {'cached_object': obj, 'meta.part_of_stream': part_of_stream, 'meta.announce_published': announce_published}}, upsert=True) - # print(f'ObjectService: {obj}') + if not reload_cache and iri in self._cache: + return self._cache[iri] + obj = get_backend().fetch_iri(iri) + self._cache[iri] = obj return obj diff --git a/utils/opengraph.py b/utils/opengraph.py index a53c07b..597ad3c 100644 --- a/utils/opengraph.py +++ b/utils/opengraph.py @@ -1,36 +1,34 @@ -from urllib.parse import urlparse - -import ipaddress import opengraph import requests from bs4 import BeautifulSoup -from .urlutils import is_url_valid, check_url +from little_boxes.urlutils import check_url +from little_boxes.urlutils import is_url_valid def links_from_note(note): - tags_href= set() - for t in note.get('tag', []): - h = t.get('href') + tags_href = set() + for t in note.get("tag", []): + h = t.get("href") if h: # TODO(tsileo): fetch the URL for Actor profile, type=mention tags_href.add(h) links = set() - soup = BeautifulSoup(note['content']) - for link in soup.find_all('a'): - h = link.get('href') - if h.startswith('http') and h not in tags_href and is_url_valid(h): + soup = BeautifulSoup(note["content"]) + for link in soup.find_all("a"): + h = link.get("href") + if h.startswith("http") and h not in tags_href and is_url_valid(h): links.add(h) return links def fetch_og_metadata(user_agent, col, remote_id): - doc = col.find_one({'remote_id': remote_id}) + doc = col.find_one({"remote_id": remote_id}) if not doc: raise ValueError - note = doc['activity']['object'] + note = doc["activity"]["object"] print(note) links = links_from_note(note) if not links: @@ -39,9 +37,11 @@ def fetch_og_metadata(user_agent, col, remote_id): htmls = [] for l in links: check_url(l) - r = requests.get(l, headers={'User-Agent': user_agent}) + r = requests.get(l, headers={"User-Agent": user_agent}) r.raise_for_status() htmls.append(r.text) links_og_metadata = [dict(opengraph.OpenGraph(html=html)) for html in htmls] - col.update_one({'remote_id': remote_id}, {'$set': {'meta.og_metadata': links_og_metadata}}) + col.update_one( + {"remote_id": remote_id}, {"$set": {"meta.og_metadata": links_og_metadata}} + ) return len(links) diff --git a/utils/urlutils.py b/utils/urlutils.py deleted file mode 100644 index 99f900d..0000000 --- a/utils/urlutils.py +++ /dev/null @@ -1,47 +0,0 @@ -import logging -import os -import socket -import ipaddress -from urllib.parse import urlparse - -from . import strtobool -from .errors import Error - -logger = logging.getLogger(__name__) - - -class InvalidURLError(Error): - pass - - -def is_url_valid(url: str) -> bool: - parsed = urlparse(url) - if parsed.scheme not in ['http', 'https']: - return False - - # XXX in debug mode, we want to allow requests to localhost to test the federation with local instances - debug_mode = strtobool(os.getenv('MICROBLOGPUB_DEBUG', 'false')) - if debug_mode: - return True - - if parsed.hostname in ['localhost']: - return False - - try: - ip_address = socket.getaddrinfo(parsed.hostname, parsed.port or 80)[0][4][0] - except socket.gaierror: - logger.exception(f'failed to lookup url {url}') - return False - - if ipaddress.ip_address(ip_address).is_private: - logger.info(f'rejecting private URL {url}') - return False - - return True - - -def check_url(url: str) -> None: - if not is_url_valid(url): - raise InvalidURLError(f'"{url}" is invalid') - - return None diff --git a/utils/webfinger.py b/utils/webfinger.py deleted file mode 100644 index 8e6fdc7..0000000 --- a/utils/webfinger.py +++ /dev/null @@ -1,75 +0,0 @@ -from urllib.parse import urlparse -from typing import Dict, Any -from typing import Optional -import logging - -import requests - -from .urlutils import check_url - - -logger = logging.getLogger(__name__) - - -def webfinger(resource: str) -> Optional[Dict[str, Any]]: - """Mastodon-like WebFinger resolution to retrieve the activity stream Actor URL. - """ - logger.info(f'performing webfinger resolution for {resource}') - protos = ['https', 'http'] - if resource.startswith('http://'): - protos.reverse() - host = urlparse(resource).netloc - elif resource.startswith('https://'): - host = urlparse(resource).netloc - else: - if resource.startswith('acct:'): - resource = resource[5:] - if resource.startswith('@'): - resource = resource[1:] - _, host = resource.split('@', 1) - resource='acct:'+resource - - # Security check on the url (like not calling localhost) - check_url(f'https://{host}') - - for i, proto in enumerate(protos): - try: - url = f'{proto}://{host}/.well-known/webfinger' - resp = requests.get( - url, - {'resource': resource} - ) - except requests.ConnectionError: - # If we tried https first and the domain is "http only" - if i == 0: - continue - break - if resp.status_code == 404: - return None - resp.raise_for_status() - return resp.json() - - -def get_remote_follow_template(resource: str) -> Optional[str]: - data = webfinger(resource) - if data is None: - return None - for link in data['links']: - if link.get('rel') == 'http://ostatus.org/schema/1.0/subscribe': - return link.get('template') - return None - - -def get_actor_url(resource: str) -> Optional[str]: - """Mastodon-like WebFinger resolution to retrieve the activity stream Actor URL. - - Returns: - the Actor URL or None if the resolution failed. - """ - data = webfinger(resource) - if data is None: - return None - for link in data['links']: - if link.get('rel') == 'self' and link.get('type') == 'application/activity+json': - return link.get('href') - return None From ad2007c1543baf1a49387f9a5c53cbb774c8c034 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 20 Jun 2018 23:42:12 +0200 Subject: [PATCH 0124/1425] Re-enable request verification --- app.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/app.py b/app.py index fd564c1..8a239ac 100644 --- a/app.py +++ b/app.py @@ -55,13 +55,13 @@ from config import custom_cache_purge_hook from little_boxes import activitypub as ap from little_boxes.activitypub import ActivityType from little_boxes.activitypub import clean_activity +from little_boxes.activitypub import get_backend from little_boxes.content_helper import parse_markdown from little_boxes.errors import ActivityNotFoundError from little_boxes.errors import Error from little_boxes.errors import NotFromOutboxError from little_boxes.httpsig import HTTPSigAuth - -# from little_boxes.httpsig import verify_request +from little_boxes.httpsig import verify_request from little_boxes.webfinger import get_actor_url from little_boxes.webfinger import get_remote_follow_template from utils.key import get_secret_key @@ -1121,15 +1121,17 @@ def inbox(): data = request.get_json(force=True) logger.debug(f"req_headers={request.headers}") logger.debug(f"raw_data={data}") - """try: - if not verify_request(ACTOR_SERVICE): + try: + if not verify_request( + request.method, request.path, request.headers, request.data + ): raise Exception("failed to verify request") except Exception: logger.exception( "failed to verify request, trying to verify the payload by fetching the remote" ) try: - data = OBJECT_SERVICE.get(data["id"]) + data = get_backend().fetch_iri(data["id"]) except Exception: logger.exception(f'failed to fetch remote id at {data["id"]}') return Response( @@ -1141,7 +1143,6 @@ def inbox(): } ), ) - """ activity = ap.parse_activity(data) logger.debug(f"inbox activity={activity}/{data}") INBOX.post(activity) From bdb25b85c224bf8f21184ab9e6a9282ff2a28298 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 22 Jun 2018 00:20:58 +0200 Subject: [PATCH 0125/1425] Update/tweak the README --- README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index fd565af..75b828a 100644 --- a/README.md +++ b/README.md @@ -23,23 +23,25 @@ - Implements [IndieAuth](https://indieauth.spec.indieweb.org/) endpoints (authorization and token endpoint) - U2F support - You can use your ActivityPub identity to login to other websites/app - - Admin UI with notifications and the stream of people you follow + - Comes with an admin UI with notifications and the stream of people you follow - Allows you to attach files to your notes - Privacy-aware image upload endpoint that strip EXIF meta data before storing the file - - No JavaScript, that's it, even the admin UI is pure HTML/CSS + - No JavaScript, **that's it**. Even the admin UI is pure HTML/CSS - Easy to customize (the theme is written Sass) - mobile-friendly theme - with dark and light version - Microformats aware (exports `h-feed`, `h-entry`, `h-cards`, ...) - - Exports RSS/Atom feeds - - Comes with a tiny HTTP API to help posting new content and performing basic actions + - Exports RSS/Atom/[JSON](https://jsonfeed.org/) feeds + - You stream/timeline is also available in an (authenticated) JSON feed + - Comes with a tiny HTTP API to help posting new content and and read your inbox/notifications - Easy to "cache" (the external/public-facing microblog part) - With a good setup, cached content can be served most of the time - You can setup a "purge" hook to let you invalidate cache when the microblog was updated - Deployable with Docker (Docker compose for everything: dev, test and deployment) - - Focus on testing - - Tested against the [official ActivityPub test suite](https://test.activitypub.rocks/) ([ ] TODO submit the report) - - CI runs some local "federation" tests + - Focused on testing + - The core ActivityPub code/tests are in [Little Boxes](https://github.com/tsileo/little-boxes) + - Tested against the [official ActivityPub test suite](https://test.activitypub.rocks/) ([report submitted](https://github.com/w3c/activitypub/issues/308)) + - CI runs "federation" tests against two instances - Manually tested against [Mastodon](https://github.com/tootsuite/mastodon) - Project is running an up-to-date instance From dae538196dc2bb905354c82e2e490f5bd61e53db Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 22 Jun 2018 00:23:05 +0200 Subject: [PATCH 0126/1425] Tweak the remote follow Previously, it was 500ing if the profile already started with an @ --- app.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index 8a239ac..dbfd372 100644 --- a/app.py +++ b/app.py @@ -364,10 +364,11 @@ def remote_follow(): return render_template("remote_follow.html") csrf.protect() + profile = request.form.get("profile") + if not profile.startswith("@"): + profile = f"@{profile}" return redirect( - get_remote_follow_template("@" + request.form.get("profile")).format( - uri=f"{USERNAME}@{DOMAIN}" - ) + get_remote_follow_template(profile).format(uri=f"{USERNAME}@{DOMAIN}") ) From e2bb700de34aaafc2dca8736d4c5e97d513f45c6 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 22 Jun 2018 00:30:47 +0200 Subject: [PATCH 0127/1425] Fix the followers icon size in the theme --- sass/base_theme.scss | 3 +-- static/css/theme.css | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sass/base_theme.scss b/sass/base_theme.scss index eba792b..0ddcfd5 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -105,8 +105,7 @@ a:hover { margin-bottom: 40px; .actor-icon { - width: 100%; - max-width:120px; + width:120px; border-radius:2px; } h3 { margin: 0; } diff --git a/static/css/theme.css b/static/css/theme.css index 6b25941..f5ea0c7 100644 --- a/static/css/theme.css +++ b/static/css/theme.css @@ -1 +1 @@ -.note-container p:first-child{margin-top:0}html,body{height:100%}body{background-color:#eee;color:#111;display:flex;flex-direction:column}.base-container{flex:1 0 auto}.footer{flex-shrink:0}a,h1,h2,h3,h4,h5,h6{color:#333}a{text-decoration:none}a:hover{text-decoration:underline}.gold{color:#1d781d}#header{margin-bottom:40px}#header .title{font-size:1.2em;padding-right:15px;color:#333}#header .title:hover{text-decoration:none}#header .subtitle-username{color:#111}#header .menu{padding:20px 0 10px 0}#header .menu ul{display:inline;list-style-type:none;padding:0}#header .menu ul li{float:left;padding-right:10px;margin-bottom:10px}#header .menu a{padding:2px 7px}#header .menu a.selected{background:#1d781d;color:#eee;border-radius:2px}#header .menu a:hover{background:#1d781d;color:#eee;text-decoration:none}#container{width:90%;max-width:720px;margin:40px auto}#container #notes{margin-top:20px}.actor-box{display:block;text-decoration:none;margin-bottom:40px}.actor-box .actor-icon{width:100%;max-width:120px;border-radius:2px}.actor-box h3{margin:0}.note{display:flex;margin-bottom:70px}.note .l{color:#333}.note .h-card{flex:initial;width:50px}.note .u-photo{width:50px;border-radius:2px}.note .note-wrapper{flex:1;padding-left:15px}.note .bottom-bar{margin-top:10px}.note .img-attachment{max-width:100%;border-radius:2px}.note h3{font-size:1.1em;color:#555}.note strong{font-weight:600}.note .note-container{clear:right;padding:10px 0}.bar-item{background:#ddd;padding:5px;color:#555;margin-right:5px;border-radius:2px}button.bar-item{border:0}form.action-form{display:inline}.perma{font-size:1.25em}.bottom-bar .perma-item{margin-right:5px}.bottom-bar a.bar-item:hover{text-decoration:none}.footer>div{width:90%;max-width:720px;margin:40px auto}.footer a,.footer a:hover,.footer a:visited{text-decoration:underline;color:#111}.summary{color:#111;font-size:1.3em;margin-top:50px;margin-bottom:70px}.summary a,.summay a:hover{color:#111;text-decoration:underline}#followers,#following,#new{margin-top:50px}#admin{margin-top:50px}textarea,input{background:#ddd;padding:10px;color:#555;border:0px;border-radius:2px}input{padding:10px}input[type=submit]{color:#1d781d;text-transform:uppercase} +.note-container p:first-child{margin-top:0}html,body{height:100%}body{background-color:#eee;color:#111;display:flex;flex-direction:column}.base-container{flex:1 0 auto}.footer{flex-shrink:0}a,h1,h2,h3,h4,h5,h6{color:#333}a{text-decoration:none}a:hover{text-decoration:underline}.gold{color:#1d781d}#header{margin-bottom:40px}#header .title{font-size:1.2em;padding-right:15px;color:#333}#header .title:hover{text-decoration:none}#header .subtitle-username{color:#111}#header .menu{padding:20px 0 10px 0}#header .menu ul{display:inline;list-style-type:none;padding:0}#header .menu ul li{float:left;padding-right:10px;margin-bottom:10px}#header .menu a{padding:2px 7px}#header .menu a.selected{background:#1d781d;color:#eee;border-radius:2px}#header .menu a:hover{background:#1d781d;color:#eee;text-decoration:none}#container{width:90%;max-width:720px;margin:40px auto}#container #notes{margin-top:20px}.actor-box{display:block;text-decoration:none;margin-bottom:40px}.actor-box .actor-icon{width:120px;border-radius:2px}.actor-box h3{margin:0}.note{display:flex;margin-bottom:70px}.note .l{color:#333}.note .h-card{flex:initial;width:50px}.note .u-photo{width:50px;border-radius:2px}.note .note-wrapper{flex:1;padding-left:15px}.note .bottom-bar{margin-top:10px}.note .img-attachment{max-width:100%;border-radius:2px}.note h3{font-size:1.1em;color:#555}.note strong{font-weight:600}.note .note-container{clear:right;padding:10px 0}.bar-item{background:#ddd;padding:5px;color:#555;margin-right:5px;border-radius:2px}button.bar-item{border:0}form.action-form{display:inline}.perma{font-size:1.25em}.bottom-bar .perma-item{margin-right:5px}.bottom-bar a.bar-item:hover{text-decoration:none}.footer>div{width:90%;max-width:720px;margin:40px auto}.footer a,.footer a:hover,.footer a:visited{text-decoration:underline;color:#111}.summary{color:#111;font-size:1.3em;margin-top:50px;margin-bottom:70px}.summary a,.summay a:hover{color:#111;text-decoration:underline}#followers,#following,#new{margin-top:50px}#admin{margin-top:50px}textarea,input{background:#ddd;padding:10px;color:#555;border:0px;border-radius:2px}input{padding:10px}input[type=submit]{color:#1d781d;text-transform:uppercase} From 63f88d41c797013958ba18924b2b4b2ba0f80b79 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 22 Jun 2018 00:43:28 +0200 Subject: [PATCH 0128/1425] Remove old webmention link in the template --- templates/layout.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/templates/layout.html b/templates/layout.html index f1cc5b0..b6f159d 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -2,14 +2,13 @@ - + {% block title %}{{ config.NAME }}{% endblock %} - microblog.pub - From 46f38e05c981aa90b6ca6b29af076bd8f0fd5350 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 22 Jun 2018 00:55:50 +0200 Subject: [PATCH 0129/1425] More docstrings --- activitypub.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/activitypub.py b/activitypub.py index 21edb4c..f54f345 100644 --- a/activitypub.py +++ b/activitypub.py @@ -52,13 +52,19 @@ def ensure_it_is_me(f): class MicroblogPubBackend(Backend): + """Implements a Little Boxes backend, backed by MongoDB.""" + def user_agent(self) -> str: + """Setup a custom user agent.""" return USER_AGENT def base_url(self) -> str: + """Base URL config.""" return BASE_URL def activity_url(self, obj_id): + """URL for activity link.""" + # FIXME(tsileo): what about note `url`? return f"{BASE_URL}/outbox/{obj_id}" @ensure_it_is_me @@ -410,6 +416,7 @@ def json_feed(path: str) -> Dict[str, Any]: def build_inbox_json_feed( path: str, request_cursor: Optional[str] = None ) -> Dict[str, Any]: + """Build a JSON feed from the inbox activities.""" data = [] cursor = None @@ -463,6 +470,7 @@ def parse_collection( def embed_collection(total_items, first_page_id): + """Helper creating a root OrderedCollection with a link to the first page.""" return { "type": ap.ActivityType.ORDERED_COLLECTION.value, "totalItems": total_items, @@ -474,6 +482,7 @@ def embed_collection(total_items, first_page_id): def build_ordered_collection( col, q=None, cursor=None, map_func=None, limit=50, col_name=None, first_page=False ): + """Helper for building an OrderedCollection from a MongoDB query (with pagination support).""" col_name = col_name or col.name if q is None: q = {} From d44387f439ef769f4eeaef70de432e5eb89335be Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 22 Jun 2018 09:35:18 +0200 Subject: [PATCH 0130/1425] More css tweaks --- sass/base_theme.scss | 8 ++++++++ static/css/theme.css | 2 +- templates/utils.html | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/sass/base_theme.scss b/sass/base_theme.scss index 0ddcfd5..a621b8d 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -103,12 +103,20 @@ a:hover { display: block; text-decoration: none; margin-bottom: 40px; + white-space: nowrap; .actor-icon { width:120px; border-radius:2px; } + h3 { margin: 0; } + + .actor-inline { + text-overflow: ellipsis; + white-space: nowrap; + overflow: hidden; + } } .note { display: flex; diff --git a/static/css/theme.css b/static/css/theme.css index f5ea0c7..d21b104 100644 --- a/static/css/theme.css +++ b/static/css/theme.css @@ -1 +1 @@ -.note-container p:first-child{margin-top:0}html,body{height:100%}body{background-color:#eee;color:#111;display:flex;flex-direction:column}.base-container{flex:1 0 auto}.footer{flex-shrink:0}a,h1,h2,h3,h4,h5,h6{color:#333}a{text-decoration:none}a:hover{text-decoration:underline}.gold{color:#1d781d}#header{margin-bottom:40px}#header .title{font-size:1.2em;padding-right:15px;color:#333}#header .title:hover{text-decoration:none}#header .subtitle-username{color:#111}#header .menu{padding:20px 0 10px 0}#header .menu ul{display:inline;list-style-type:none;padding:0}#header .menu ul li{float:left;padding-right:10px;margin-bottom:10px}#header .menu a{padding:2px 7px}#header .menu a.selected{background:#1d781d;color:#eee;border-radius:2px}#header .menu a:hover{background:#1d781d;color:#eee;text-decoration:none}#container{width:90%;max-width:720px;margin:40px auto}#container #notes{margin-top:20px}.actor-box{display:block;text-decoration:none;margin-bottom:40px}.actor-box .actor-icon{width:120px;border-radius:2px}.actor-box h3{margin:0}.note{display:flex;margin-bottom:70px}.note .l{color:#333}.note .h-card{flex:initial;width:50px}.note .u-photo{width:50px;border-radius:2px}.note .note-wrapper{flex:1;padding-left:15px}.note .bottom-bar{margin-top:10px}.note .img-attachment{max-width:100%;border-radius:2px}.note h3{font-size:1.1em;color:#555}.note strong{font-weight:600}.note .note-container{clear:right;padding:10px 0}.bar-item{background:#ddd;padding:5px;color:#555;margin-right:5px;border-radius:2px}button.bar-item{border:0}form.action-form{display:inline}.perma{font-size:1.25em}.bottom-bar .perma-item{margin-right:5px}.bottom-bar a.bar-item:hover{text-decoration:none}.footer>div{width:90%;max-width:720px;margin:40px auto}.footer a,.footer a:hover,.footer a:visited{text-decoration:underline;color:#111}.summary{color:#111;font-size:1.3em;margin-top:50px;margin-bottom:70px}.summary a,.summay a:hover{color:#111;text-decoration:underline}#followers,#following,#new{margin-top:50px}#admin{margin-top:50px}textarea,input{background:#ddd;padding:10px;color:#555;border:0px;border-radius:2px}input{padding:10px}input[type=submit]{color:#1d781d;text-transform:uppercase} +.note-container p:first-child{margin-top:0}html,body{height:100%}body{background-color:#eee;color:#111;display:flex;flex-direction:column}.base-container{flex:1 0 auto}.footer{flex-shrink:0}a,h1,h2,h3,h4,h5,h6{color:#333}a{text-decoration:none}a:hover{text-decoration:underline}.gold{color:#1d781d}#header{margin-bottom:40px}#header .title{font-size:1.2em;padding-right:15px;color:#333}#header .title:hover{text-decoration:none}#header .subtitle-username{color:#111}#header .menu{padding:20px 0 10px 0}#header .menu ul{display:inline;list-style-type:none;padding:0}#header .menu ul li{float:left;padding-right:10px;margin-bottom:10px}#header .menu a{padding:2px 7px}#header .menu a.selected{background:#1d781d;color:#eee;border-radius:2px}#header .menu a:hover{background:#1d781d;color:#eee;text-decoration:none}#container{width:90%;max-width:720px;margin:40px auto}#container #notes{margin-top:20px}.actor-box{display:block;text-decoration:none;margin-bottom:40px;white-space:nowrap}.actor-box .actor-icon{width:120px;border-radius:2px}.actor-box h3{margin:0}.actor-box .actor-inline{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.note{display:flex;margin-bottom:70px}.note .l{color:#333}.note .h-card{flex:initial;width:50px}.note .u-photo{width:50px;border-radius:2px}.note .note-wrapper{flex:1;padding-left:15px}.note .bottom-bar{margin-top:10px}.note .img-attachment{max-width:100%;border-radius:2px}.note h3{font-size:1.1em;color:#555}.note strong{font-weight:600}.note .note-container{clear:right;padding:10px 0}.bar-item{background:#ddd;padding:5px;color:#555;margin-right:5px;border-radius:2px}button.bar-item{border:0}form.action-form{display:inline}.perma{font-size:1.25em}.bottom-bar .perma-item{margin-right:5px}.bottom-bar a.bar-item:hover{text-decoration:none}.footer>div{width:90%;max-width:720px;margin:40px auto}.footer a,.footer a:hover,.footer a:visited{text-decoration:underline;color:#111}.summary{color:#111;font-size:1.3em;margin-top:50px;margin-bottom:70px}.summary a,.summay a:hover{color:#111;text-decoration:underline}#followers,#following,#new{margin-top:50px}#admin{margin-top:50px}textarea,input{background:#ddd;padding:10px;color:#555;border:0px;border-radius:2px}input{padding:10px}input[type=submit]{color:#1d781d;text-transform:uppercase} diff --git a/templates/utils.html b/templates/utils.html index 4d7ce89..d04fed0 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -6,7 +6,7 @@ {% else %} {% endif %} -
    +
    {{ follower.name or follower.preferredUsername }}
    @{{ follower.preferredUsername }}@{{ follower.url | domain }}
    From 0e981459897128940a9f53e1159ce28cfdd901b6 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 22 Jun 2018 10:01:05 +0200 Subject: [PATCH 0131/1425] Fix the followers/following page CSS --- sass/base_theme.scss | 16 +++++++++++++++- static/css/theme.css | 2 +- templates/utils.html | 4 ++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/sass/base_theme.scss b/sass/base_theme.scss index a621b8d..775019a 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -103,7 +103,6 @@ a:hover { display: block; text-decoration: none; margin-bottom: 40px; - white-space: nowrap; .actor-icon { width:120px; @@ -118,6 +117,21 @@ a:hover { overflow: hidden; } } +.actor-box { + display: block; + text-decoration: none; + + .actor-box-wrapper { + margin-bottom:40px; + + .actor-icon { + width:120px; + border-radius:2px; + } + + h3 { margin: 0; } + } +} .note { display: flex; margin-bottom: 70px; diff --git a/static/css/theme.css b/static/css/theme.css index d21b104..65dc70b 100644 --- a/static/css/theme.css +++ b/static/css/theme.css @@ -1 +1 @@ -.note-container p:first-child{margin-top:0}html,body{height:100%}body{background-color:#eee;color:#111;display:flex;flex-direction:column}.base-container{flex:1 0 auto}.footer{flex-shrink:0}a,h1,h2,h3,h4,h5,h6{color:#333}a{text-decoration:none}a:hover{text-decoration:underline}.gold{color:#1d781d}#header{margin-bottom:40px}#header .title{font-size:1.2em;padding-right:15px;color:#333}#header .title:hover{text-decoration:none}#header .subtitle-username{color:#111}#header .menu{padding:20px 0 10px 0}#header .menu ul{display:inline;list-style-type:none;padding:0}#header .menu ul li{float:left;padding-right:10px;margin-bottom:10px}#header .menu a{padding:2px 7px}#header .menu a.selected{background:#1d781d;color:#eee;border-radius:2px}#header .menu a:hover{background:#1d781d;color:#eee;text-decoration:none}#container{width:90%;max-width:720px;margin:40px auto}#container #notes{margin-top:20px}.actor-box{display:block;text-decoration:none;margin-bottom:40px;white-space:nowrap}.actor-box .actor-icon{width:120px;border-radius:2px}.actor-box h3{margin:0}.actor-box .actor-inline{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.note{display:flex;margin-bottom:70px}.note .l{color:#333}.note .h-card{flex:initial;width:50px}.note .u-photo{width:50px;border-radius:2px}.note .note-wrapper{flex:1;padding-left:15px}.note .bottom-bar{margin-top:10px}.note .img-attachment{max-width:100%;border-radius:2px}.note h3{font-size:1.1em;color:#555}.note strong{font-weight:600}.note .note-container{clear:right;padding:10px 0}.bar-item{background:#ddd;padding:5px;color:#555;margin-right:5px;border-radius:2px}button.bar-item{border:0}form.action-form{display:inline}.perma{font-size:1.25em}.bottom-bar .perma-item{margin-right:5px}.bottom-bar a.bar-item:hover{text-decoration:none}.footer>div{width:90%;max-width:720px;margin:40px auto}.footer a,.footer a:hover,.footer a:visited{text-decoration:underline;color:#111}.summary{color:#111;font-size:1.3em;margin-top:50px;margin-bottom:70px}.summary a,.summay a:hover{color:#111;text-decoration:underline}#followers,#following,#new{margin-top:50px}#admin{margin-top:50px}textarea,input{background:#ddd;padding:10px;color:#555;border:0px;border-radius:2px}input{padding:10px}input[type=submit]{color:#1d781d;text-transform:uppercase} +.note-container p:first-child{margin-top:0}html,body{height:100%}body{background-color:#eee;color:#111;display:flex;flex-direction:column}.base-container{flex:1 0 auto}.footer{flex-shrink:0}a,h1,h2,h3,h4,h5,h6{color:#333}a{text-decoration:none}a:hover{text-decoration:underline}.gold{color:#1d781d}#header{margin-bottom:40px}#header .title{font-size:1.2em;padding-right:15px;color:#333}#header .title:hover{text-decoration:none}#header .subtitle-username{color:#111}#header .menu{padding:20px 0 10px 0}#header .menu ul{display:inline;list-style-type:none;padding:0}#header .menu ul li{float:left;padding-right:10px;margin-bottom:10px}#header .menu a{padding:2px 7px}#header .menu a.selected{background:#1d781d;color:#eee;border-radius:2px}#header .menu a:hover{background:#1d781d;color:#eee;text-decoration:none}#container{width:90%;max-width:720px;margin:40px auto}#container #notes{margin-top:20px}.actor-box{display:block;text-decoration:none;margin-bottom:40px}.actor-box .actor-icon{width:120px;border-radius:2px}.actor-box h3{margin:0}.actor-box .actor-inline{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.actor-box{display:block;text-decoration:none}.actor-box .actor-box-wrapper{margin-bottom:40px}.actor-box .actor-box-wrapper .actor-icon{width:120px;border-radius:2px}.actor-box .actor-box-wrapper h3{margin:0}.note{display:flex;margin-bottom:70px}.note .l{color:#333}.note .h-card{flex:initial;width:50px}.note .u-photo{width:50px;border-radius:2px}.note .note-wrapper{flex:1;padding-left:15px}.note .bottom-bar{margin-top:10px}.note .img-attachment{max-width:100%;border-radius:2px}.note h3{font-size:1.1em;color:#555}.note strong{font-weight:600}.note .note-container{clear:right;padding:10px 0}.bar-item{background:#ddd;padding:5px;color:#555;margin-right:5px;border-radius:2px}button.bar-item{border:0}form.action-form{display:inline}.perma{font-size:1.25em}.bottom-bar .perma-item{margin-right:5px}.bottom-bar a.bar-item:hover{text-decoration:none}.footer>div{width:90%;max-width:720px;margin:40px auto}.footer a,.footer a:hover,.footer a:visited{text-decoration:underline;color:#111}.summary{color:#111;font-size:1.3em;margin-top:50px;margin-bottom:70px}.summary a,.summay a:hover{color:#111;text-decoration:underline}#followers,#following,#new{margin-top:50px}#admin{margin-top:50px}textarea,input{background:#ddd;padding:10px;color:#555;border:0px;border-radius:2px}input{padding:10px}input[type=submit]{color:#1d781d;text-transform:uppercase} diff --git a/templates/utils.html b/templates/utils.html index d04fed0..299fd1e 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -15,8 +15,8 @@ {% macro display_actor(follower) -%} - -
    + +
    {% if not follower.icon %} From 01c7edc5179e9f0b542321c51e8dabcf6476dc86 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 22 Jun 2018 22:27:28 +0200 Subject: [PATCH 0132/1425] Add badges in the menu, tweak the design --- app.py | 35 +++++++++++++++++++++++------------ requirements.txt | 1 + sass/base_theme.scss | 31 ++++++++++++++++++++++--------- static/css/theme.css | 2 +- templates/header.html | 10 +++++----- 5 files changed, 52 insertions(+), 27 deletions(-) diff --git a/app.py b/app.py index dbfd372..722e16b 100644 --- a/app.py +++ b/app.py @@ -6,6 +6,7 @@ import mimetypes import os import urllib from datetime import datetime +from datetime import timezone from functools import wraps from typing import Any from typing import Dict @@ -18,6 +19,7 @@ import piexif import pymongo import timeago from bson.objectid import ObjectId +from dateutil import parser from flask import Flask from flask import Response from flask import abort @@ -103,10 +105,27 @@ def verify_pass(pwd): @app.context_processor def inject_config(): + q = { + "type": "Create", + "activity.object.type": "Note", + "activity.object.inReplyTo": None, + "meta.deleted": False, + } + notes_count = DB.outbox.find( + {"$or": [q, {"type": "Announce", "meta.undo": False}]} + ).count() + q = {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} + with_replies_count = DB.outbox.find( + {"$or": [q, {"type": "Announce", "meta.undo": False}]} + ).count() return dict( microblogpub_version=VERSION, config=config, logged_in=session.get("logged_in", False), + followers_count=DB.followers.count(), + following_count=DB.following.count(), + notes_count=notes_count, + with_replies_count=with_replies_count, ) @@ -183,24 +202,16 @@ def get_actor(url): @app.template_filter() def format_time(val): if val: - return datetime.strftime( - datetime.strptime(val, "%Y-%m-%dT%H:%M:%SZ"), "%B %d, %Y, %H:%M %p" - ) + dt = parser.parse(val) + return datetime.strftime(dt, "%B %d, %Y, %H:%M %p") return val @app.template_filter() def format_timeago(val): if val: - try: - return timeago.format( - datetime.strptime(val, "%Y-%m-%dT%H:%M:%SZ"), datetime.utcnow() - ) - except Exception: - return timeago.format( - datetime.strptime(val, "%Y-%m-%dT%H:%M:%S.%fZ"), datetime.utcnow() - ) - + dt = parser.parse(val) + return timeago.format(dt, datetime.now(timezone.utc)) return val diff --git a/requirements.txt b/requirements.txt index eb16141..3c3b9a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +python-dateutil libsass gunicorn piexif diff --git a/sass/base_theme.scss b/sass/base_theme.scss index 775019a..00174ac 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -77,7 +77,15 @@ a:hover { } } a { - padding: 2px 7px; + padding: 5px 10px; + small.badge { + background-color: $color-menu-background; + color: $color-light; + border-radius: 2px; + margin-left: 5px; + padding: 3px 5px 0px 5px; + font-weight: bold; + } } a.selected { background: $primary-color; @@ -88,6 +96,11 @@ a:hover { background: $primary-color; color: $background-color; text-decoration: none; + border-radius: 2px; + .badge { + color: $primary-color; + background: $background-color; + } } } } @@ -105,7 +118,7 @@ a:hover { margin-bottom: 40px; .actor-icon { - width:120px; + width: 120px; border-radius:2px; } @@ -117,7 +130,7 @@ a:hover { overflow: hidden; } } -.actor-box { +.actor-box-big { display: block; text-decoration: none; @@ -206,19 +219,19 @@ form.action-form { } .summary { color: $color-summary; - font-size:1.3em; - margin-top:50px; - margin-bottom:70px; + font-size: 1.3em; + margin-top: 60px; + margin-bottom: 50px; } .summary a, .summay a:hover { color: $color-summary; - text-decoration:underline; + text-decoration: underline; } #followers, #following, #new { - margin-top:50px; + margin-top: 50px; } #admin { - margin-top:50px; + margin-top: 50px; } textarea, input { background: $color-menu-background; diff --git a/static/css/theme.css b/static/css/theme.css index 65dc70b..a4e9946 100644 --- a/static/css/theme.css +++ b/static/css/theme.css @@ -1 +1 @@ -.note-container p:first-child{margin-top:0}html,body{height:100%}body{background-color:#eee;color:#111;display:flex;flex-direction:column}.base-container{flex:1 0 auto}.footer{flex-shrink:0}a,h1,h2,h3,h4,h5,h6{color:#333}a{text-decoration:none}a:hover{text-decoration:underline}.gold{color:#1d781d}#header{margin-bottom:40px}#header .title{font-size:1.2em;padding-right:15px;color:#333}#header .title:hover{text-decoration:none}#header .subtitle-username{color:#111}#header .menu{padding:20px 0 10px 0}#header .menu ul{display:inline;list-style-type:none;padding:0}#header .menu ul li{float:left;padding-right:10px;margin-bottom:10px}#header .menu a{padding:2px 7px}#header .menu a.selected{background:#1d781d;color:#eee;border-radius:2px}#header .menu a:hover{background:#1d781d;color:#eee;text-decoration:none}#container{width:90%;max-width:720px;margin:40px auto}#container #notes{margin-top:20px}.actor-box{display:block;text-decoration:none;margin-bottom:40px}.actor-box .actor-icon{width:120px;border-radius:2px}.actor-box h3{margin:0}.actor-box .actor-inline{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.actor-box{display:block;text-decoration:none}.actor-box .actor-box-wrapper{margin-bottom:40px}.actor-box .actor-box-wrapper .actor-icon{width:120px;border-radius:2px}.actor-box .actor-box-wrapper h3{margin:0}.note{display:flex;margin-bottom:70px}.note .l{color:#333}.note .h-card{flex:initial;width:50px}.note .u-photo{width:50px;border-radius:2px}.note .note-wrapper{flex:1;padding-left:15px}.note .bottom-bar{margin-top:10px}.note .img-attachment{max-width:100%;border-radius:2px}.note h3{font-size:1.1em;color:#555}.note strong{font-weight:600}.note .note-container{clear:right;padding:10px 0}.bar-item{background:#ddd;padding:5px;color:#555;margin-right:5px;border-radius:2px}button.bar-item{border:0}form.action-form{display:inline}.perma{font-size:1.25em}.bottom-bar .perma-item{margin-right:5px}.bottom-bar a.bar-item:hover{text-decoration:none}.footer>div{width:90%;max-width:720px;margin:40px auto}.footer a,.footer a:hover,.footer a:visited{text-decoration:underline;color:#111}.summary{color:#111;font-size:1.3em;margin-top:50px;margin-bottom:70px}.summary a,.summay a:hover{color:#111;text-decoration:underline}#followers,#following,#new{margin-top:50px}#admin{margin-top:50px}textarea,input{background:#ddd;padding:10px;color:#555;border:0px;border-radius:2px}input{padding:10px}input[type=submit]{color:#1d781d;text-transform:uppercase} +.note-container p:first-child{margin-top:0}html,body{height:100%}body{background-color:#eee;color:#111;display:flex;flex-direction:column}.base-container{flex:1 0 auto}.footer{flex-shrink:0}a,h1,h2,h3,h4,h5,h6{color:#333}a{text-decoration:none}a:hover{text-decoration:underline}.gold{color:#1d781d}#header{margin-bottom:40px}#header .title{font-size:1.2em;padding-right:15px;color:#333}#header .title:hover{text-decoration:none}#header .subtitle-username{color:#111}#header .menu{padding:20px 0 10px 0}#header .menu ul{display:inline;list-style-type:none;padding:0}#header .menu ul li{float:left;padding-right:10px;margin-bottom:10px}#header .menu a{padding:5px 10px}#header .menu a small.badge{background-color:#ddd;color:#555;border-radius:2px;margin-left:5px;padding:3px 5px 0px 5px;font-weight:bold}#header .menu a.selected{background:#1d781d;color:#eee;border-radius:2px}#header .menu a:hover{background:#1d781d;color:#eee;text-decoration:none;border-radius:2px}#header .menu a:hover .badge{color:#1d781d;background:#eee}#container{width:90%;max-width:720px;margin:40px auto}#container #notes{margin-top:20px}.actor-box{display:block;text-decoration:none;margin-bottom:40px}.actor-box .actor-icon{width:120px;border-radius:2px}.actor-box h3{margin:0}.actor-box .actor-inline{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.actor-box-big{display:block;text-decoration:none}.actor-box-big .actor-box-wrapper{margin-bottom:40px}.actor-box-big .actor-box-wrapper .actor-icon{width:120px;border-radius:2px}.actor-box-big .actor-box-wrapper h3{margin:0}.note{display:flex;margin-bottom:70px}.note .l{color:#333}.note .h-card{flex:initial;width:50px}.note .u-photo{width:50px;border-radius:2px}.note .note-wrapper{flex:1;padding-left:15px}.note .bottom-bar{margin-top:10px}.note .img-attachment{max-width:100%;border-radius:2px}.note h3{font-size:1.1em;color:#555}.note strong{font-weight:600}.note .note-container{clear:right;padding:10px 0}.bar-item{background:#ddd;padding:5px;color:#555;margin-right:5px;border-radius:2px}button.bar-item{border:0}form.action-form{display:inline}.perma{font-size:1.25em}.bottom-bar .perma-item{margin-right:5px}.bottom-bar a.bar-item:hover{text-decoration:none}.footer>div{width:90%;max-width:720px;margin:40px auto}.footer a,.footer a:hover,.footer a:visited{text-decoration:underline;color:#111}.summary{color:#111;font-size:1.3em;margin-top:60px;margin-bottom:50px}.summary a,.summay a:hover{color:#111;text-decoration:underline}#followers,#following,#new{margin-top:50px}#admin{margin-top:50px}textarea,input{background:#ddd;padding:10px;color:#555;border:0px;border-radius:2px}input{padding:10px}input[type=submit]{color:#1d781d;text-transform:uppercase} diff --git a/templates/header.html b/templates/header.html index 9983dca..e6ea410 100644 --- a/templates/header.html +++ b/templates/header.html @@ -10,10 +10,10 @@
    From 49ddb955cdb121979b1c832e5bdbb7e33cbfae35 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 22 Jun 2018 22:51:11 +0200 Subject: [PATCH 0133/1425] More CSS imrovements --- sass/base_theme.scss | 9 +++++++-- static/css/theme.css | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/sass/base_theme.scss b/sass/base_theme.scss index 00174ac..40aa8b1 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -51,7 +51,7 @@ a:hover { } #header { - margin-bottom: 40px; + margin-bottom: 60px; .title { font-size: 1.2em; @@ -91,6 +91,11 @@ a:hover { background: $primary-color; color: $background-color; border-radius:2px; + .badge { + color: $primary-color; + background: $background-color; + } + } a:hover { background: $primary-color; @@ -220,7 +225,7 @@ form.action-form { .summary { color: $color-summary; font-size: 1.3em; - margin-top: 60px; + margin-top: 10px; margin-bottom: 50px; } .summary a, .summay a:hover { diff --git a/static/css/theme.css b/static/css/theme.css index a4e9946..edf3a94 100644 --- a/static/css/theme.css +++ b/static/css/theme.css @@ -1 +1 @@ -.note-container p:first-child{margin-top:0}html,body{height:100%}body{background-color:#eee;color:#111;display:flex;flex-direction:column}.base-container{flex:1 0 auto}.footer{flex-shrink:0}a,h1,h2,h3,h4,h5,h6{color:#333}a{text-decoration:none}a:hover{text-decoration:underline}.gold{color:#1d781d}#header{margin-bottom:40px}#header .title{font-size:1.2em;padding-right:15px;color:#333}#header .title:hover{text-decoration:none}#header .subtitle-username{color:#111}#header .menu{padding:20px 0 10px 0}#header .menu ul{display:inline;list-style-type:none;padding:0}#header .menu ul li{float:left;padding-right:10px;margin-bottom:10px}#header .menu a{padding:5px 10px}#header .menu a small.badge{background-color:#ddd;color:#555;border-radius:2px;margin-left:5px;padding:3px 5px 0px 5px;font-weight:bold}#header .menu a.selected{background:#1d781d;color:#eee;border-radius:2px}#header .menu a:hover{background:#1d781d;color:#eee;text-decoration:none;border-radius:2px}#header .menu a:hover .badge{color:#1d781d;background:#eee}#container{width:90%;max-width:720px;margin:40px auto}#container #notes{margin-top:20px}.actor-box{display:block;text-decoration:none;margin-bottom:40px}.actor-box .actor-icon{width:120px;border-radius:2px}.actor-box h3{margin:0}.actor-box .actor-inline{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.actor-box-big{display:block;text-decoration:none}.actor-box-big .actor-box-wrapper{margin-bottom:40px}.actor-box-big .actor-box-wrapper .actor-icon{width:120px;border-radius:2px}.actor-box-big .actor-box-wrapper h3{margin:0}.note{display:flex;margin-bottom:70px}.note .l{color:#333}.note .h-card{flex:initial;width:50px}.note .u-photo{width:50px;border-radius:2px}.note .note-wrapper{flex:1;padding-left:15px}.note .bottom-bar{margin-top:10px}.note .img-attachment{max-width:100%;border-radius:2px}.note h3{font-size:1.1em;color:#555}.note strong{font-weight:600}.note .note-container{clear:right;padding:10px 0}.bar-item{background:#ddd;padding:5px;color:#555;margin-right:5px;border-radius:2px}button.bar-item{border:0}form.action-form{display:inline}.perma{font-size:1.25em}.bottom-bar .perma-item{margin-right:5px}.bottom-bar a.bar-item:hover{text-decoration:none}.footer>div{width:90%;max-width:720px;margin:40px auto}.footer a,.footer a:hover,.footer a:visited{text-decoration:underline;color:#111}.summary{color:#111;font-size:1.3em;margin-top:60px;margin-bottom:50px}.summary a,.summay a:hover{color:#111;text-decoration:underline}#followers,#following,#new{margin-top:50px}#admin{margin-top:50px}textarea,input{background:#ddd;padding:10px;color:#555;border:0px;border-radius:2px}input{padding:10px}input[type=submit]{color:#1d781d;text-transform:uppercase} +.note-container p:first-child{margin-top:0}html,body{height:100%}body{background-color:#eee;color:#111;display:flex;flex-direction:column}.base-container{flex:1 0 auto}.footer{flex-shrink:0}a,h1,h2,h3,h4,h5,h6{color:#333}a{text-decoration:none}a:hover{text-decoration:underline}.gold{color:#1d781d}#header{margin-bottom:60px}#header .title{font-size:1.2em;padding-right:15px;color:#333}#header .title:hover{text-decoration:none}#header .subtitle-username{color:#111}#header .menu{padding:20px 0 10px 0}#header .menu ul{display:inline;list-style-type:none;padding:0}#header .menu ul li{float:left;padding-right:10px;margin-bottom:10px}#header .menu a{padding:5px 10px}#header .menu a small.badge{background-color:#ddd;color:#555;border-radius:2px;margin-left:5px;padding:3px 5px 0px 5px;font-weight:bold}#header .menu a.selected{background:#1d781d;color:#eee;border-radius:2px}#header .menu a.selected .badge{color:#1d781d;background:#eee}#header .menu a:hover{background:#1d781d;color:#eee;text-decoration:none;border-radius:2px}#header .menu a:hover .badge{color:#1d781d;background:#eee}#container{width:90%;max-width:720px;margin:40px auto}#container #notes{margin-top:20px}.actor-box{display:block;text-decoration:none;margin-bottom:40px}.actor-box .actor-icon{width:120px;border-radius:2px}.actor-box h3{margin:0}.actor-box .actor-inline{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.actor-box-big{display:block;text-decoration:none}.actor-box-big .actor-box-wrapper{margin-bottom:40px}.actor-box-big .actor-box-wrapper .actor-icon{width:120px;border-radius:2px}.actor-box-big .actor-box-wrapper h3{margin:0}.note{display:flex;margin-bottom:70px}.note .l{color:#333}.note .h-card{flex:initial;width:50px}.note .u-photo{width:50px;border-radius:2px}.note .note-wrapper{flex:1;padding-left:15px}.note .bottom-bar{margin-top:10px}.note .img-attachment{max-width:100%;border-radius:2px}.note h3{font-size:1.1em;color:#555}.note strong{font-weight:600}.note .note-container{clear:right;padding:10px 0}.bar-item{background:#ddd;padding:5px;color:#555;margin-right:5px;border-radius:2px}button.bar-item{border:0}form.action-form{display:inline}.perma{font-size:1.25em}.bottom-bar .perma-item{margin-right:5px}.bottom-bar a.bar-item:hover{text-decoration:none}.footer>div{width:90%;max-width:720px;margin:40px auto}.footer a,.footer a:hover,.footer a:visited{text-decoration:underline;color:#111}.summary{color:#111;font-size:1.3em;margin-top:10px;margin-bottom:50px}.summary a,.summay a:hover{color:#111;text-decoration:underline}#followers,#following,#new{margin-top:50px}#admin{margin-top:50px}textarea,input{background:#ddd;padding:10px;color:#555;border:0px;border-radius:2px}input{padding:10px}input[type=submit]{color:#1d781d;text-transform:uppercase} From 200c6edc1809051b81d734fdaf56c03c0f001873 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 23 Jun 2018 00:29:06 +0200 Subject: [PATCH 0134/1425] Better threads handling --- activitypub.py | 75 ++++++++++++++++++++++++++++++++++++++++++++------ config.py | 2 +- 2 files changed, 67 insertions(+), 10 deletions(-) diff --git a/activitypub.py b/activitypub.py index f54f345..ae7970b 100644 --- a/activitypub.py +++ b/activitypub.py @@ -241,10 +241,15 @@ class MicroblogPubBackend(Backend): @ensure_it_is_me def inbox_delete(self, as_actor: ap.Person, delete: ap.Delete) -> None: - DB.inbox.update_one( + if not DB.inbox.find_one_and_update( {"activity.object.id": delete.get_object().id}, {"$set": {"meta.deleted": True}}, - ) + ): + DB.threads.update_one( + {"activity.object.id": delete.get_object().id}, + {"$set": {"meta.deleted": True}}, + ) + obj = delete.get_object() if obj.ACTIVITY_TYPE != ap.ActivityType.NOTE: obj = ap.parse_activity( @@ -290,11 +295,14 @@ class MicroblogPubBackend(Backend): def inbox_update(self, as_actor: ap.Person, update: ap.Update) -> None: obj = update.get_object() if obj.ACTIVITY_TYPE == ap.ActivityType.NOTE: - DB.inbox.update_one( + if not DB.inbox.find_one_and_update( {"activity.object.id": obj.id}, {"$set": {"activity.object": obj.to_dict()}}, - ) - return + ): + DB.threads.update_one( + {"activity.object.id": obj.id}, + {"$set": {"activity.object": obj.to_dict()}}, + ) # FIXME(tsileo): handle update actor amd inbox_update_note/inbox_update_actor @@ -342,25 +350,74 @@ class MicroblogPubBackend(Backend): {"activity.object.id": in_reply_to}, {"$inc": {"meta.count_reply": -1, "meta.count_direct_reply": -1}}, ): - DB.outbox.update_one( + if not DB.outbox.find_one_and_update( {"activity.object.id": in_reply_to}, {"$inc": {"meta.count_reply": -1, "meta.count_direct_reply": -1}}, - ) + ): + DB.threads.update_one( + {"activity.object.id": in_reply_to}, + {"$inc": {"meta.count_reply": -1, "meta.count_direct_reply": -1}}, + ) @ensure_it_is_me def _handle_replies(self, as_actor: ap.Person, create: ap.Create) -> None: + """Go up to the root reply, store unknown replies in the `threads` DB and set the "meta.thread_root_parent" + key to make it easy to query a whole thread.""" in_reply_to = create.get_object().inReplyTo if not in_reply_to: pass + new_threads = [] + root_reply = in_reply_to + reply = ap.fetch_remote_activity(root_reply, expected=ap.ActivityType.NOTE) + if not DB.inbox.find_one_and_update( {"activity.object.id": in_reply_to}, {"$inc": {"meta.count_reply": 1, "meta.count_direct_reply": 1}}, ): - DB.outbox.update_one( + if not DB.outbox.find_one_and_update( {"activity.object.id": in_reply_to}, {"$inc": {"meta.count_reply": 1, "meta.count_direct_reply": 1}}, - ) + ): + # It means the activity is not in the inbox, and not in the outbox, we want to save it + DB.threads.insert_one( + { + "activity": reply.to_dict(), + "type": reply.type, + "remote_id": reply.id, + "meta": {"undo": False, "deleted": False}, + } + ) + new_threads.append(reply.id) + + while reply is not None: + in_reply_to = reply.inReplyTo + if not in_reply_to: + break + root_reply = in_reply_to + reply = ap.fetch_remote_activity(root_reply, expected=ap.ActivityType.NOTE) + q = {"activity.object.id": root_reply} + if not DB.inbox.count(q) and not DB.outbox.count(q): + DB.threads.insert_one( + { + "activity": reply.to_dict(), + "type": reply.type, + "remote_id": reply.id, + "meta": {"undo": False, "deleted": False}, + } + ) + new_threads.append(reply.id) + + q = {"remote_id": create.id} + if not DB.inbox.find_one_and_update( + q, {"$set": {"meta.thread_root_parent": root_reply}} + ): + DB.outbox.update_one(q, {"$set": {"meta.thread_root_parent": root_reply}}) + + DB.threads.update( + {"remote_id": {"$in": new_threads}}, + {"$set": {"meta.thread_root_parent": root_reply}}, + ) def gen_feed(): diff --git a/config.py b/config.py index 44659c1..1a00bb7 100644 --- a/config.py +++ b/config.py @@ -58,7 +58,7 @@ with open(os.path.join(KEY_DIR, "me.yml")) as f: THEME_COLOR = conf.get("theme_color") USER_AGENT = ( - f"{requests.utils.default_user_agent()} " f"(microblog.pub/{VERSION}; +{BASE_URL})" + f"{requests.utils.default_user_agent()} (microblog.pub/{VERSION}; +{BASE_URL})" ) mongo_client = MongoClient( From be682080563518dc3c0351fb9d5be874a2915a41 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 23 Jun 2018 01:04:58 +0200 Subject: [PATCH 0135/1425] Bugfixes --- activitypub.py | 2 +- app.py | 27 ++++++++++++++++----------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/activitypub.py b/activitypub.py index ae7970b..a167360 100644 --- a/activitypub.py +++ b/activitypub.py @@ -365,7 +365,7 @@ class MicroblogPubBackend(Backend): key to make it easy to query a whole thread.""" in_reply_to = create.get_object().inReplyTo if not in_reply_to: - pass + return new_threads = [] root_reply = in_reply_to diff --git a/app.py b/app.py index 722e16b..ca9448b 100644 --- a/app.py +++ b/app.py @@ -516,29 +516,34 @@ def with_replies(): def _build_thread(data, include_children=True): data["_requested"] = True + print(data) root_id = data["meta"].get("thread_root_parent", data["activity"]["object"]["id"]) - thread_ids = data["meta"].get("thread_parents", []) - if include_children: - thread_ids.extend(data["meta"].get("thread_children", [])) + query = {"$or": [{"meta.thread_root_parent": root_id, "type": "Create"}]} + if data['activity']['object'].get('inReplyTo'): + query['$or'].append({'activity.object.id': data['activity']['object']['inReplyTo']}) - query = { - "activity.object.id": {"$in": thread_ids}, - "type": "Create", - "meta.deleted": False, # TODO(tsileo): handle Tombstone instead of filtering them - } # Fetch the root replies, and the children - replies = [data] + list(DB.inbox.find(query)) + list(DB.outbox.find(query)) - + replies = ( + [data] + + list(DB.inbox.find(query)) + + list(DB.outbox.find(query)) + + list(DB.threads.find(query)) + ) + replies = sorted(replies, key=lambda d: d["activity"]["object"]["published"]) # Index all the IDs in order to build a tree idx = {} + replies2 = [] for rep in replies: rep_id = rep["activity"]["object"]["id"] + if rep_id in idx: + continue idx[rep_id] = rep.copy() idx[rep_id]["_nodes"] = [] + replies2.append(rep) # Build the tree - for rep in replies: + for rep in replies2: rep_id = rep["activity"]["object"]["id"] if rep_id == root_id: continue From 5ce3d57faf862f53945fd19f83cef37ec78b846d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 23 Jun 2018 01:14:35 +0200 Subject: [PATCH 0136/1425] Fix the note URL issue --- activitypub.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/activitypub.py b/activitypub.py index a167360..d5a61ed 100644 --- a/activitypub.py +++ b/activitypub.py @@ -64,9 +64,12 @@ class MicroblogPubBackend(Backend): def activity_url(self, obj_id): """URL for activity link.""" - # FIXME(tsileo): what about note `url`? return f"{BASE_URL}/outbox/{obj_id}" + def note_url(self, obj_id): + """URL for activity link.""" + return f"{BASE_URL}/note/{obj_id}" + @ensure_it_is_me def outbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: DB.outbox.insert_one( From a43a946f5d67b32c2101427f8cb68a15c77fb4a5 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 23 Jun 2018 12:01:31 +0200 Subject: [PATCH 0137/1425] Fix the follower/following page layout --- templates/followers.html | 18 +++++++++++++++++- templates/utils.html | 12 ++++++------ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/templates/followers.html b/templates/followers.html index e4f43a1..4ad731b 100644 --- a/templates/followers.html +++ b/templates/followers.html @@ -9,8 +9,24 @@
    {% for follower in followers_data %} - {{ utils.display_actor(follower) }} +
    + {{ utils.display_actor_inline(follower, size=80) }} +
    {% endfor %} + + {% for follower in followers_data %} +
    + {{ utils.display_actor_inline(follower, size=80) }} +
    + {% endfor %} + + {% for follower in followers_data %} +
    + {{ utils.display_actor_inline(follower, size=80) }} +
    + {% endfor %} + +
    diff --git a/templates/utils.html b/templates/utils.html index 299fd1e..5bd6091 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -1,10 +1,10 @@ -{% macro display_actor_inline(follower) -%} +{% macro display_actor_inline(follower, size=50) -%} {% if not follower.icon %} - + {% else %} -{% endif %} +{% endif %}
    {{ follower.name or follower.preferredUsername }}
    @@ -80,9 +80,9 @@ {% else %}     permalink -{% if item.meta.count_reply %}{{ item.meta.count_reply }} replies{% endif %} -{% if item.meta.count_boost %}{{ item.meta.count_boost }} boosts{% endif %} -{% if item.meta.count_like %}{{ item.meta.count_like }} likes{% endif %} +{% if item.meta.count_reply %}{{ item.meta.count_reply }} replies{% endif %} +{% if item.meta.count_boost %}{{ item.meta.count_boost }} boosts{% endif %} +{% if item.meta.count_like %}{{ item.meta.count_like }} likes{% endif %} {% endif %} {% if ui and session.logged_in %} From 4969b9feb99d6626bc51bab67b35b141c70a4c81 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 23 Jun 2018 12:18:02 +0200 Subject: [PATCH 0138/1425] Fix template --- templates/followers.html | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/templates/followers.html b/templates/followers.html index 4ad731b..e08ad2a 100644 --- a/templates/followers.html +++ b/templates/followers.html @@ -13,20 +13,6 @@ {{ utils.display_actor_inline(follower, size=80) }}
    {% endfor %} - - {% for follower in followers_data %} -
    - {{ utils.display_actor_inline(follower, size=80) }} -
    - {% endfor %} - - {% for follower in followers_data %} -
    - {{ utils.display_actor_inline(follower, size=80) }} -
    - {% endfor %} - -
    From db45233ebc78ab738d458b8a6fb3ba2021cdd1a7 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 23 Jun 2018 12:19:34 +0200 Subject: [PATCH 0139/1425] Tweak the Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 925bc09..846d967 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ reload-fed: WEB_PORT=5007 CONFIG_DIR=./tests/fixtures/instance2/config docker-compose -p instance2 -f docker-compose-tests.yml up -d --force-recreate --build update: - docker-compose stop git pull docker build . -t microblogpub:latest + docker-compose stop docker-compose up -d --force-recreate --build From c8eed9e89c89e4d13c6f12afb0368e61834de8a6 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 23 Jun 2018 20:52:35 +0200 Subject: [PATCH 0140/1425] Update README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 75b828a..570d139 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@

    Build Status + License

    From c85fb8530e74151694624505ad4dc62863b16066 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 24 Jun 2018 19:22:40 +0200 Subject: [PATCH 0141/1425] Fix remote follow (fixes #10) --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index ca9448b..0ca4da6 100644 --- a/app.py +++ b/app.py @@ -397,7 +397,7 @@ def authorize_follow(): if DB.following.find({"remote_actor": actor}).count() > 0: return redirect("/following") - follow = activitypub.Follow(actor=MY_PERSON.id, object=actor) + follow = ap.Follow(actor=MY_PERSON.id, object=actor) OUTBOX.post(follow) return redirect("/following") From 38e3cc6bb7790b563f94c2880d62efb1a02fabca Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 24 Jun 2018 19:51:09 +0200 Subject: [PATCH 0142/1425] Fix the debug mode --- activitypub.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/activitypub.py b/activitypub.py index d5a61ed..0b0d8a3 100644 --- a/activitypub.py +++ b/activitypub.py @@ -1,4 +1,5 @@ import logging +import os from datetime import datetime from typing import Any from typing import Dict @@ -17,6 +18,7 @@ from config import ID from config import ME from config import USER_AGENT from config import USERNAME +from little_boxes import strtobool from little_boxes import activitypub as ap from little_boxes.backend import Backend from little_boxes.collection import parse_collection as ap_parse_collection @@ -54,6 +56,9 @@ def ensure_it_is_me(f): class MicroblogPubBackend(Backend): """Implements a Little Boxes backend, backed by MongoDB.""" + def debug_mode(self) -> bool: + return strtobool(os.getenv("MICROBLOGPUB_DEBUG", "false")) + def user_agent(self) -> str: """Setup a custom user agent.""" return USER_AGENT From 2bb4cadc931bd00f57f75eea977bbafb3347f8d9 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 25 Jun 2018 23:45:43 +0200 Subject: [PATCH 0143/1425] Tweaked the header CSS, started to work on the liked public page --- app.py | 25 ++++++++++++++++++++++--- sass/base_theme.scss | 7 ++++--- static/css/theme.css | 2 +- templates/header.html | 10 ++++++---- templates/index.html | 6 +----- templates/utils.html | 2 +- 6 files changed, 35 insertions(+), 17 deletions(-) diff --git a/app.py b/app.py index 0ca4da6..0519652 100644 --- a/app.py +++ b/app.py @@ -118,6 +118,11 @@ def inject_config(): with_replies_count = DB.outbox.find( {"$or": [q, {"type": "Announce", "meta.undo": False}]} ).count() + liked_count = DB.outbox.count({ + "meta.deleted": False, + "meta.undo": False, + "type": ActivityType.LIKE.value, + }) return dict( microblogpub_version=VERSION, config=config, @@ -125,6 +130,7 @@ def inject_config(): followers_count=DB.followers.count(), following_count=DB.following.count(), notes_count=notes_count, + liked_count=liked_count, with_replies_count=with_replies_count, ) @@ -520,8 +526,10 @@ def _build_thread(data, include_children=True): root_id = data["meta"].get("thread_root_parent", data["activity"]["object"]["id"]) query = {"$or": [{"meta.thread_root_parent": root_id, "type": "Create"}]} - if data['activity']['object'].get('inReplyTo'): - query['$or'].append({'activity.object.id': data['activity']['object']['inReplyTo']}) + if data["activity"]["object"].get("inReplyTo"): + query["$or"].append( + {"activity.object.id": data["activity"]["object"]["inReplyTo"]} + ) # Fetch the root replies, and the children replies = ( @@ -1411,7 +1419,18 @@ def tags(tag): @app.route("/liked") def liked(): if not is_api_request(): - abort(404) + return render_template( + "liked.html", + me=ME, + liked=DB.outbox.find( + { + "type": ActivityType.LIKE.value, + "meta.deleted": False, + "meta.undo": False, + } + ), + ) + q = {"meta.deleted": False, "meta.undo": False, "type": ActivityType.LIKE.value} return jsonify( **activitypub.build_ordered_collection( diff --git a/sass/base_theme.scss b/sass/base_theme.scss index 40aa8b1..b6bd26a 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -51,7 +51,7 @@ a:hover { } #header { - margin-bottom: 60px; + margin-bottom: 70px; .title { font-size: 1.2em; @@ -65,7 +65,8 @@ a:hover { color: $color; } .menu { - padding: 20px 0 10px 0; + clear: both; + padding: 0 0 10px 0; ul { display: inline; list-style-type: none; @@ -226,7 +227,7 @@ form.action-form { color: $color-summary; font-size: 1.3em; margin-top: 10px; - margin-bottom: 50px; + margin-bottom: 30px; } .summary a, .summay a:hover { color: $color-summary; diff --git a/static/css/theme.css b/static/css/theme.css index edf3a94..9c16c40 100644 --- a/static/css/theme.css +++ b/static/css/theme.css @@ -1 +1 @@ -.note-container p:first-child{margin-top:0}html,body{height:100%}body{background-color:#eee;color:#111;display:flex;flex-direction:column}.base-container{flex:1 0 auto}.footer{flex-shrink:0}a,h1,h2,h3,h4,h5,h6{color:#333}a{text-decoration:none}a:hover{text-decoration:underline}.gold{color:#1d781d}#header{margin-bottom:60px}#header .title{font-size:1.2em;padding-right:15px;color:#333}#header .title:hover{text-decoration:none}#header .subtitle-username{color:#111}#header .menu{padding:20px 0 10px 0}#header .menu ul{display:inline;list-style-type:none;padding:0}#header .menu ul li{float:left;padding-right:10px;margin-bottom:10px}#header .menu a{padding:5px 10px}#header .menu a small.badge{background-color:#ddd;color:#555;border-radius:2px;margin-left:5px;padding:3px 5px 0px 5px;font-weight:bold}#header .menu a.selected{background:#1d781d;color:#eee;border-radius:2px}#header .menu a.selected .badge{color:#1d781d;background:#eee}#header .menu a:hover{background:#1d781d;color:#eee;text-decoration:none;border-radius:2px}#header .menu a:hover .badge{color:#1d781d;background:#eee}#container{width:90%;max-width:720px;margin:40px auto}#container #notes{margin-top:20px}.actor-box{display:block;text-decoration:none;margin-bottom:40px}.actor-box .actor-icon{width:120px;border-radius:2px}.actor-box h3{margin:0}.actor-box .actor-inline{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.actor-box-big{display:block;text-decoration:none}.actor-box-big .actor-box-wrapper{margin-bottom:40px}.actor-box-big .actor-box-wrapper .actor-icon{width:120px;border-radius:2px}.actor-box-big .actor-box-wrapper h3{margin:0}.note{display:flex;margin-bottom:70px}.note .l{color:#333}.note .h-card{flex:initial;width:50px}.note .u-photo{width:50px;border-radius:2px}.note .note-wrapper{flex:1;padding-left:15px}.note .bottom-bar{margin-top:10px}.note .img-attachment{max-width:100%;border-radius:2px}.note h3{font-size:1.1em;color:#555}.note strong{font-weight:600}.note .note-container{clear:right;padding:10px 0}.bar-item{background:#ddd;padding:5px;color:#555;margin-right:5px;border-radius:2px}button.bar-item{border:0}form.action-form{display:inline}.perma{font-size:1.25em}.bottom-bar .perma-item{margin-right:5px}.bottom-bar a.bar-item:hover{text-decoration:none}.footer>div{width:90%;max-width:720px;margin:40px auto}.footer a,.footer a:hover,.footer a:visited{text-decoration:underline;color:#111}.summary{color:#111;font-size:1.3em;margin-top:10px;margin-bottom:50px}.summary a,.summay a:hover{color:#111;text-decoration:underline}#followers,#following,#new{margin-top:50px}#admin{margin-top:50px}textarea,input{background:#ddd;padding:10px;color:#555;border:0px;border-radius:2px}input{padding:10px}input[type=submit]{color:#1d781d;text-transform:uppercase} +.note-container p:first-child{margin-top:0}html,body{height:100%}body{background-color:#eee;color:#111;display:flex;flex-direction:column}.base-container{flex:1 0 auto}.footer{flex-shrink:0}a,h1,h2,h3,h4,h5,h6{color:#333}a{text-decoration:none}a:hover{text-decoration:underline}.gold{color:#1d781d}#header{margin-bottom:70px}#header .title{font-size:1.2em;padding-right:15px;color:#333}#header .title:hover{text-decoration:none}#header .subtitle-username{color:#111}#header .menu{clear:both;padding:0 0 10px 0}#header .menu ul{display:inline;list-style-type:none;padding:0}#header .menu ul li{float:left;padding-right:10px;margin-bottom:10px}#header .menu a{padding:5px 10px}#header .menu a small.badge{background-color:#ddd;color:#555;border-radius:2px;margin-left:5px;padding:3px 5px 0px 5px;font-weight:bold}#header .menu a.selected{background:#1d781d;color:#eee;border-radius:2px}#header .menu a.selected .badge{color:#1d781d;background:#eee}#header .menu a:hover{background:#1d781d;color:#eee;text-decoration:none;border-radius:2px}#header .menu a:hover .badge{color:#1d781d;background:#eee}#container{width:90%;max-width:720px;margin:40px auto}#container #notes{margin-top:20px}.actor-box{display:block;text-decoration:none;margin-bottom:40px}.actor-box .actor-icon{width:120px;border-radius:2px}.actor-box h3{margin:0}.actor-box .actor-inline{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.actor-box-big{display:block;text-decoration:none}.actor-box-big .actor-box-wrapper{margin-bottom:40px}.actor-box-big .actor-box-wrapper .actor-icon{width:120px;border-radius:2px}.actor-box-big .actor-box-wrapper h3{margin:0}.note{display:flex;margin-bottom:70px}.note .l{color:#333}.note .h-card{flex:initial;width:50px}.note .u-photo{width:50px;border-radius:2px}.note .note-wrapper{flex:1;padding-left:15px}.note .bottom-bar{margin-top:10px}.note .img-attachment{max-width:100%;border-radius:2px}.note h3{font-size:1.1em;color:#555}.note strong{font-weight:600}.note .note-container{clear:right;padding:10px 0}.bar-item{background:#ddd;padding:5px;color:#555;margin-right:5px;border-radius:2px}button.bar-item{border:0}form.action-form{display:inline}.perma{font-size:1.25em}.bottom-bar .perma-item{margin-right:5px}.bottom-bar a.bar-item:hover{text-decoration:none}.footer>div{width:90%;max-width:720px;margin:40px auto}.footer a,.footer a:hover,.footer a:visited{text-decoration:underline;color:#111}.summary{color:#111;font-size:1.3em;margin-top:10px;margin-bottom:30px}.summary a,.summay a:hover{color:#111;text-decoration:underline}#followers,#following,#new{margin-top:50px}#admin{margin-top:50px}textarea,input{background:#ddd;padding:10px;color:#555;border:0px;border-radius:2px}input{padding:10px}input[type=submit]{color:#1d781d;text-transform:uppercase} diff --git a/templates/header.html b/templates/header.html index e6ea410..5f23c2e 100644 --- a/templates/header.html +++ b/templates/header.html @@ -1,17 +1,21 @@ From 0780bf3691d93c33b9fc4666dfdaa95e4da2eed0 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 26 Jun 2018 22:52:19 +0200 Subject: [PATCH 0145/1425] More design tweaks, fix the index Fixes #12 --- app.py | 6 ++++-- sass/base_theme.scss | 2 +- static/css/theme.css | 2 +- templates/header.html | 1 - 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app.py b/app.py index 0519652..3b947dc 100644 --- a/app.py +++ b/app.py @@ -457,14 +457,16 @@ def index(): for data in outbox_data: if data["type"] == "Announce": - print(data) if data["activity"]["object"].startswith("http"): data["ref"] = { "activity": { - "object": OBJECT_SERVICE.get(data["activity"]["object"]) + "object": OBJECT_SERVICE.get(data["activity"]["object"]), + "id": "NA", }, "meta": {}, } + print(data) + return render_template( "index.html", diff --git a/sass/base_theme.scss b/sass/base_theme.scss index b6bd26a..1cbe524 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -225,7 +225,7 @@ form.action-form { } .summary { color: $color-summary; - font-size: 1.3em; + font-size: 1.1em; margin-top: 10px; margin-bottom: 30px; } diff --git a/static/css/theme.css b/static/css/theme.css index 9c16c40..138747f 100644 --- a/static/css/theme.css +++ b/static/css/theme.css @@ -1 +1 @@ -.note-container p:first-child{margin-top:0}html,body{height:100%}body{background-color:#eee;color:#111;display:flex;flex-direction:column}.base-container{flex:1 0 auto}.footer{flex-shrink:0}a,h1,h2,h3,h4,h5,h6{color:#333}a{text-decoration:none}a:hover{text-decoration:underline}.gold{color:#1d781d}#header{margin-bottom:70px}#header .title{font-size:1.2em;padding-right:15px;color:#333}#header .title:hover{text-decoration:none}#header .subtitle-username{color:#111}#header .menu{clear:both;padding:0 0 10px 0}#header .menu ul{display:inline;list-style-type:none;padding:0}#header .menu ul li{float:left;padding-right:10px;margin-bottom:10px}#header .menu a{padding:5px 10px}#header .menu a small.badge{background-color:#ddd;color:#555;border-radius:2px;margin-left:5px;padding:3px 5px 0px 5px;font-weight:bold}#header .menu a.selected{background:#1d781d;color:#eee;border-radius:2px}#header .menu a.selected .badge{color:#1d781d;background:#eee}#header .menu a:hover{background:#1d781d;color:#eee;text-decoration:none;border-radius:2px}#header .menu a:hover .badge{color:#1d781d;background:#eee}#container{width:90%;max-width:720px;margin:40px auto}#container #notes{margin-top:20px}.actor-box{display:block;text-decoration:none;margin-bottom:40px}.actor-box .actor-icon{width:120px;border-radius:2px}.actor-box h3{margin:0}.actor-box .actor-inline{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.actor-box-big{display:block;text-decoration:none}.actor-box-big .actor-box-wrapper{margin-bottom:40px}.actor-box-big .actor-box-wrapper .actor-icon{width:120px;border-radius:2px}.actor-box-big .actor-box-wrapper h3{margin:0}.note{display:flex;margin-bottom:70px}.note .l{color:#333}.note .h-card{flex:initial;width:50px}.note .u-photo{width:50px;border-radius:2px}.note .note-wrapper{flex:1;padding-left:15px}.note .bottom-bar{margin-top:10px}.note .img-attachment{max-width:100%;border-radius:2px}.note h3{font-size:1.1em;color:#555}.note strong{font-weight:600}.note .note-container{clear:right;padding:10px 0}.bar-item{background:#ddd;padding:5px;color:#555;margin-right:5px;border-radius:2px}button.bar-item{border:0}form.action-form{display:inline}.perma{font-size:1.25em}.bottom-bar .perma-item{margin-right:5px}.bottom-bar a.bar-item:hover{text-decoration:none}.footer>div{width:90%;max-width:720px;margin:40px auto}.footer a,.footer a:hover,.footer a:visited{text-decoration:underline;color:#111}.summary{color:#111;font-size:1.3em;margin-top:10px;margin-bottom:30px}.summary a,.summay a:hover{color:#111;text-decoration:underline}#followers,#following,#new{margin-top:50px}#admin{margin-top:50px}textarea,input{background:#ddd;padding:10px;color:#555;border:0px;border-radius:2px}input{padding:10px}input[type=submit]{color:#1d781d;text-transform:uppercase} +.note-container p:first-child{margin-top:0}html,body{height:100%}body{background-color:#eee;color:#111;display:flex;flex-direction:column}.base-container{flex:1 0 auto}.footer{flex-shrink:0}a,h1,h2,h3,h4,h5,h6{color:#333}a{text-decoration:none}a:hover{text-decoration:underline}.gold{color:#1d781d}#header{margin-bottom:70px}#header .title{font-size:1.2em;padding-right:15px;color:#333}#header .title:hover{text-decoration:none}#header .subtitle-username{color:#111}#header .menu{clear:both;padding:0 0 10px 0}#header .menu ul{display:inline;list-style-type:none;padding:0}#header .menu ul li{float:left;padding-right:10px;margin-bottom:10px}#header .menu a{padding:5px 10px}#header .menu a small.badge{background-color:#ddd;color:#555;border-radius:2px;margin-left:5px;padding:3px 5px 0px 5px;font-weight:bold}#header .menu a.selected{background:#1d781d;color:#eee;border-radius:2px}#header .menu a.selected .badge{color:#1d781d;background:#eee}#header .menu a:hover{background:#1d781d;color:#eee;text-decoration:none;border-radius:2px}#header .menu a:hover .badge{color:#1d781d;background:#eee}#container{width:90%;max-width:720px;margin:40px auto}#container #notes{margin-top:20px}.actor-box{display:block;text-decoration:none;margin-bottom:40px}.actor-box .actor-icon{width:120px;border-radius:2px}.actor-box h3{margin:0}.actor-box .actor-inline{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.actor-box-big{display:block;text-decoration:none}.actor-box-big .actor-box-wrapper{margin-bottom:40px}.actor-box-big .actor-box-wrapper .actor-icon{width:120px;border-radius:2px}.actor-box-big .actor-box-wrapper h3{margin:0}.note{display:flex;margin-bottom:70px}.note .l{color:#333}.note .h-card{flex:initial;width:50px}.note .u-photo{width:50px;border-radius:2px}.note .note-wrapper{flex:1;padding-left:15px}.note .bottom-bar{margin-top:10px}.note .img-attachment{max-width:100%;border-radius:2px}.note h3{font-size:1.1em;color:#555}.note strong{font-weight:600}.note .note-container{clear:right;padding:10px 0}.bar-item{background:#ddd;padding:5px;color:#555;margin-right:5px;border-radius:2px}button.bar-item{border:0}form.action-form{display:inline}.perma{font-size:1.25em}.bottom-bar .perma-item{margin-right:5px}.bottom-bar a.bar-item:hover{text-decoration:none}.footer>div{width:90%;max-width:720px;margin:40px auto}.footer a,.footer a:hover,.footer a:visited{text-decoration:underline;color:#111}.summary{color:#111;font-size:1.1em;margin-top:10px;margin-bottom:30px}.summary a,.summay a:hover{color:#111;text-decoration:underline}#followers,#following,#new{margin-top:50px}#admin{margin-top:50px}textarea,input{background:#ddd;padding:10px;color:#555;border:0px;border-radius:2px}input{padding:10px}input[type=submit]{color:#1d781d;text-transform:uppercase} diff --git a/templates/header.html b/templates/header.html index c9ed818..167cc1d 100644 --- a/templates/header.html +++ b/templates/header.html @@ -27,7 +27,6 @@
  • /admin
  • /logout
    • {% endif %} -
  • About
    • From 19974360157dad1944ac63a7c128f0e48a99457a Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 26 Jun 2018 23:42:36 +0200 Subject: [PATCH 0146/1425] Mention the matrix room in the README (#microblog.pub:matrix.org) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 570d139..5d11155 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@

    Build Status - +#microblog.pub on Matrix License

    From 541bf1c63beee962ad941d04e02acf11ad2f2c0e Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 27 Jun 2018 23:27:48 +0200 Subject: [PATCH 0147/1425] Fix the CSS for the following page --- README.md | 2 +- activitypub.py | 7 ------- templates/following.html | 4 +++- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 5d11155..56897d1 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@

    Build Status -#microblog.pub on Matrix +#microblog.pub on Matrix License

    diff --git a/activitypub.py b/activitypub.py index 0b0d8a3..dd86e6c 100644 --- a/activitypub.py +++ b/activitypub.py @@ -273,13 +273,6 @@ class MicroblogPubBackend(Backend): if obj: self._handle_replies_delete(as_actor, obj) - # FIXME(tsileo): handle threads - # obj = delete._get_actual_object() - # if obj.type_enum == ActivityType.NOTE: - # obj._delete_from_threads() - - # TODO(tsileo): also purge the cache if it's a reply of a published activity - @ensure_it_is_me def outbox_delete(self, as_actor: ap.Person, delete: ap.Delete) -> None: DB.outbox.update_one( diff --git a/templates/following.html b/templates/following.html index 83e8b0d..0e6efb4 100644 --- a/templates/following.html +++ b/templates/following.html @@ -9,7 +9,9 @@
    {% for followed in following_data %} - {{ utils.display_actor(followed) }} +
    + {{ utils.display_actor_inline(followed, size=80) }} +
    {% endfor %}
    From 0c2030605dda7f966cf648757ab8df8648630b1a Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 28 Jun 2018 00:37:18 +0200 Subject: [PATCH 0148/1425] Better activity type handling --- activitypub.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/activitypub.py b/activitypub.py index dd86e6c..e27dfd7 100644 --- a/activitypub.py +++ b/activitypub.py @@ -5,7 +5,6 @@ from typing import Any from typing import Dict from typing import List from typing import Optional -from typing import Union from bson.objectid import ObjectId from feedgen.feed import FeedGenerator @@ -23,6 +22,7 @@ from little_boxes import activitypub as ap from little_boxes.backend import Backend from little_boxes.collection import parse_collection as ap_parse_collection from little_boxes.errors import Error +from little_boxes.activitypub import _to_list logger = logging.getLogger(__name__) @@ -35,13 +35,6 @@ def _remove_id(doc: ap.ObjectType) -> ap.ObjectType: return doc -def _to_list(data: Union[List[Any], Any]) -> List[Any]: - """Helper to convert fields that can be either an object or a list of objects to a list of object.""" - if isinstance(data, list): - return data - return [data] - - def ensure_it_is_me(f): """Method decorator used to track the events fired during tests.""" @@ -80,7 +73,7 @@ class MicroblogPubBackend(Backend): DB.outbox.insert_one( { "activity": activity.to_dict(), - "type": activity.type, + "type": _to_list(activity.type), "remote_id": activity.id, "meta": {"undo": False, "deleted": False}, } @@ -131,7 +124,7 @@ class MicroblogPubBackend(Backend): DB.inbox.insert_one( { "activity": activity.to_dict(), - "type": activity.type, + "type": _to_list(activity.type), "remote_id": activity.id, "meta": {"undo": False, "deleted": False}, } @@ -384,7 +377,7 @@ class MicroblogPubBackend(Backend): DB.threads.insert_one( { "activity": reply.to_dict(), - "type": reply.type, + "type": _to_list(reply.type), "remote_id": reply.id, "meta": {"undo": False, "deleted": False}, } @@ -402,7 +395,7 @@ class MicroblogPubBackend(Backend): DB.threads.insert_one( { "activity": reply.to_dict(), - "type": reply.type, + "type": _to_list(reply.type), "remote_id": reply.id, "meta": {"undo": False, "deleted": False}, } From eb4d795ede595728c99283681e2b9f0077e539bc Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 29 Jun 2018 22:16:26 +0200 Subject: [PATCH 0149/1425] HUGE cleanup --- activitypub.py | 207 ++++++++++------------ app.py | 402 +++++++++++++++++++++--------------------- templates/index.html | 13 +- templates/liked.html | 18 ++ templates/stream.html | 30 ++-- templates/tags.html | 2 +- templates/utils.html | 80 ++++----- 7 files changed, 370 insertions(+), 382 deletions(-) create mode 100644 templates/liked.html diff --git a/activitypub.py b/activitypub.py index e27dfd7..b891674 100644 --- a/activitypub.py +++ b/activitypub.py @@ -1,6 +1,7 @@ import logging import os from datetime import datetime +from enum import Enum from typing import Any from typing import Dict from typing import List @@ -17,12 +18,12 @@ from config import ID from config import ME from config import USER_AGENT from config import USERNAME -from little_boxes import strtobool from little_boxes import activitypub as ap +from little_boxes import strtobool +from little_boxes.activitypub import _to_list from little_boxes.backend import Backend from little_boxes.collection import parse_collection as ap_parse_collection from little_boxes.errors import Error -from little_boxes.activitypub import _to_list logger = logging.getLogger(__name__) @@ -46,6 +47,12 @@ def ensure_it_is_me(f): return wrapper +class Box(Enum): + INBOX = "inbox" + OUTBOX = "outbox" + REPLIES = "replies" + + class MicroblogPubBackend(Backend): """Implements a Little Boxes backend, backed by MongoDB.""" @@ -68,10 +75,11 @@ class MicroblogPubBackend(Backend): """URL for activity link.""" return f"{BASE_URL}/note/{obj_id}" - @ensure_it_is_me - def outbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: - DB.outbox.insert_one( + def save(self, box: Box, activity: ap.BaseActivity) -> None: + """Custom helper for saving an activity to the DB.""" + DB.activities.insert_one( { + "box": box.value, "activity": activity.to_dict(), "type": _to_list(activity.type), "remote_id": activity.id, @@ -79,11 +87,16 @@ class MicroblogPubBackend(Backend): } ) + @ensure_it_is_me + def outbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: + self.save(Box.OUTBOX, activity) + @ensure_it_is_me def outbox_is_blocked(self, as_actor: ap.Person, actor_id: str) -> bool: return bool( - DB.outbox.find_one( + DB.activities.find_one( { + "box": Box.OUTBOX.value, "type": ap.ActivityType.BLOCK.value, "activity.object": actor_id, "meta.undo": False, @@ -101,14 +114,14 @@ class MicroblogPubBackend(Backend): if iri.endswith("/activity"): iri = iri.replace("/activity", "") is_a_note = True - data = DB.outbox.find_one({"remote_id": iri}) - if data: - if is_a_note: - return data["activity"]["object"] + data = DB.activities.find_one({"box": Box.OUTBOX.value, "remote_id": iri}) + if data and is_a_note: + return data["activity"]["object"] + elif data: return data["activity"] else: # Check if the activity is stored in the inbox - data = DB.inbox.find_one({"remote_id": iri}) + data = DB.activities.find_one({"remote_id": iri}) if data: return data["activity"] @@ -117,18 +130,11 @@ class MicroblogPubBackend(Backend): @ensure_it_is_me def inbox_check_duplicate(self, as_actor: ap.Person, iri: str) -> bool: - return bool(DB.inbox.find_one({"remote_id": iri})) + return bool(DB.activities.find_one({"box": Box.INBOX.value, "remote_id": iri})) @ensure_it_is_me def inbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: - DB.inbox.insert_one( - { - "activity": activity.to_dict(), - "type": _to_list(activity.type), - "remote_id": activity.id, - "meta": {"undo": False, "deleted": False}, - } - ) + self.save(Box.INBOX, activity) @ensure_it_is_me def post_to_remote_inbox(self, as_actor: ap.Person, payload: str, to: str) -> None: @@ -161,41 +167,37 @@ class MicroblogPubBackend(Backend): def inbox_like(self, as_actor: ap.Person, like: ap.Like) -> None: obj = like.get_object() # Update the meta counter if the object is published by the server - DB.outbox.update_one( - {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": 1}} + DB.activities.update_one( + {"box": Box.OUTBOX.value, "activity.object.id": obj.id}, + {"$inc": {"meta.count_like": 1}}, ) @ensure_it_is_me def inbox_undo_like(self, as_actor: ap.Person, like: ap.Like) -> None: obj = like.get_object() # Update the meta counter if the object is published by the server - DB.outbox.update_one( - {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": -1}} + DB.activities.update_one( + {"box": Box.OUTBOX.value, "activity.object.id": obj.id}, + {"$inc": {"meta.count_like": -1}}, ) @ensure_it_is_me def outbox_like(self, as_actor: ap.Person, like: ap.Like) -> None: obj = like.get_object() - # Unlikely, but an actor can like it's own post - DB.outbox.update_one( - {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": 1}} + DB.activities.update_one( + {"activity.object.id": obj.id}, + {"$inc": {"meta.count_like": 1}, "$set": {"meta.liked": like.id}}, ) - - # Keep track of the like we just performed - DB.inbox.update_one( - {"activity.object.id": obj.id}, {"$set": {"meta.liked": like.id}} + DB.activities.update_one( + {"remote_id": like.id}, {"$set": {"meta.object": obj.to_dict(embed=True)}} ) @ensure_it_is_me def outbox_undo_like(self, as_actor: ap.Person, like: ap.Like) -> None: obj = like.get_object() - # Unlikely, but an actor can like it's own post - DB.outbox.update_one( - {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": -1}} - ) - - DB.inbox.update_one( - {"activity.object.id": obj.id}, {"$set": {"meta.liked": False}} + DB.activities.update_one( + {"activity.object.id": obj.id}, + {"$inc": {"meta.count_like": -1}, "$set": {"meta.liked": False}}, ) @ensure_it_is_me @@ -204,57 +206,57 @@ class MicroblogPubBackend(Backend): "object" ].startswith("http"): # TODO(tsileo): actually drop it without storing it and better logging, also move the check somewhere else + # or remote it? logger.warn( f'received an Annouce referencing an OStatus notice ({announce._data["object"]}), dropping the message' ) return - # FIXME(tsileo): Save/cache the object, and make it part of the stream so we can fetch it - if isinstance(announce._data["object"], str): - obj_iri = announce._data["object"] - else: - obj_iri = self.get_object().id - DB.outbox.update_one( - {"activity.object.id": obj_iri}, {"$inc": {"meta.count_boost": 1}} + obj = announce.get_object() + DB.activities.update_one( + {"remote_id": announce.id}, + {"$set": {"meta.object": obj.to_dict(embed=True)}}, + ) + DB.activities.update_one( + {"activity.object.id": obj.id}, {"$inc": {"meta.count_boost": 1}} ) @ensure_it_is_me def inbox_undo_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: obj = announce.get_object() # Update the meta counter if the object is published by the server - DB.outbox.update_one( + DB.activities.update_one( {"activity.object.id": obj.id}, {"$inc": {"meta.count_boost": -1}} ) @ensure_it_is_me def outbox_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: obj = announce.get_object() - DB.inbox.update_one( + DB.activities.update_one( + {"remote_id": announce.id}, + {"$set": {"meta.object": obj.to_dict(embed=True)}}, + ) + DB.activities.update_one( {"activity.object.id": obj.id}, {"$set": {"meta.boosted": announce.id}} ) @ensure_it_is_me def outbox_undo_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: obj = announce.get_object() - DB.inbox.update_one( + DB.activities.update_one( {"activity.object.id": obj.id}, {"$set": {"meta.boosted": False}} ) @ensure_it_is_me def inbox_delete(self, as_actor: ap.Person, delete: ap.Delete) -> None: - if not DB.inbox.find_one_and_update( - {"activity.object.id": delete.get_object().id}, - {"$set": {"meta.deleted": True}}, - ): - DB.threads.update_one( - {"activity.object.id": delete.get_object().id}, - {"$set": {"meta.deleted": True}}, - ) - obj = delete.get_object() + DB.activities.update_one( + {"activity.object.id": obj.id}, {"$set": {"meta.deleted": True}} + ) + if obj.ACTIVITY_TYPE != ap.ActivityType.NOTE: obj = ap.parse_activity( - DB.inbox.find_one( + DB.activities.find_one( { "activity.object.id": delete.get_object().id, "type": ap.ActivityType.CREATE.value, @@ -268,14 +270,14 @@ class MicroblogPubBackend(Backend): @ensure_it_is_me def outbox_delete(self, as_actor: ap.Person, delete: ap.Delete) -> None: - DB.outbox.update_one( + DB.activities.update_one( {"activity.object.id": delete.get_object().id}, {"$set": {"meta.deleted": True}}, ) obj = delete.get_object() if delete.get_object().ACTIVITY_TYPE != ap.ActivityType.NOTE: obj = ap.parse_activity( - DB.outbox.find_one( + DB.activities.find_one( { "activity.object.id": delete.get_object().id, "type": ap.ActivityType.CREATE.value, @@ -289,15 +291,10 @@ class MicroblogPubBackend(Backend): def inbox_update(self, as_actor: ap.Person, update: ap.Update) -> None: obj = update.get_object() if obj.ACTIVITY_TYPE == ap.ActivityType.NOTE: - if not DB.inbox.find_one_and_update( + DB.activities.update_one( {"activity.object.id": obj.id}, {"$set": {"activity.object": obj.to_dict()}}, - ): - DB.threads.update_one( - {"activity.object.id": obj.id}, - {"$set": {"activity.object": obj.to_dict()}}, - ) - + ) # FIXME(tsileo): handle update actor amd inbox_update_note/inbox_update_actor @ensure_it_is_me @@ -322,7 +319,7 @@ class MicroblogPubBackend(Backend): print(f"updating note from outbox {obj!r} {update}") logger.info(f"updating note from outbox {obj!r} {update}") - DB.outbox.update_one({"activity.object.id": obj["id"]}, update) + DB.activities.update_one({"activity.object.id": obj["id"]}, update) # FIXME(tsileo): should send an Update (but not a partial one, to all the note's recipients # (create a new Update with the result of the update, and send it without saving it?) @@ -340,18 +337,10 @@ class MicroblogPubBackend(Backend): if not in_reply_to: pass - if not DB.inbox.find_one_and_update( + DB.activities.update_one( {"activity.object.id": in_reply_to}, {"$inc": {"meta.count_reply": -1, "meta.count_direct_reply": -1}}, - ): - if not DB.outbox.find_one_and_update( - {"activity.object.id": in_reply_to}, - {"$inc": {"meta.count_reply": -1, "meta.count_direct_reply": -1}}, - ): - DB.threads.update_one( - {"activity.object.id": in_reply_to}, - {"$inc": {"meta.count_reply": -1, "meta.count_direct_reply": -1}}, - ) + ) @ensure_it_is_me def _handle_replies(self, as_actor: ap.Person, create: ap.Create) -> None: @@ -365,24 +354,14 @@ class MicroblogPubBackend(Backend): root_reply = in_reply_to reply = ap.fetch_remote_activity(root_reply, expected=ap.ActivityType.NOTE) - if not DB.inbox.find_one_and_update( + creply = DB.activities.find_one_and_update( {"activity.object.id": in_reply_to}, {"$inc": {"meta.count_reply": 1, "meta.count_direct_reply": 1}}, - ): - if not DB.outbox.find_one_and_update( - {"activity.object.id": in_reply_to}, - {"$inc": {"meta.count_reply": 1, "meta.count_direct_reply": 1}}, - ): - # It means the activity is not in the inbox, and not in the outbox, we want to save it - DB.threads.insert_one( - { - "activity": reply.to_dict(), - "type": _to_list(reply.type), - "remote_id": reply.id, - "meta": {"undo": False, "deleted": False}, - } - ) - new_threads.append(reply.id) + ) + if not creply: + # It means the activity is not in the inbox, and not in the outbox, we want to save it + self.save(Box.REPLIES, reply) + new_threads.append(reply.id) while reply is not None: in_reply_to = reply.inReplyTo @@ -391,25 +370,15 @@ class MicroblogPubBackend(Backend): root_reply = in_reply_to reply = ap.fetch_remote_activity(root_reply, expected=ap.ActivityType.NOTE) q = {"activity.object.id": root_reply} - if not DB.inbox.count(q) and not DB.outbox.count(q): - DB.threads.insert_one( - { - "activity": reply.to_dict(), - "type": _to_list(reply.type), - "remote_id": reply.id, - "meta": {"undo": False, "deleted": False}, - } - ) + if not DB.activities.count(q): + self.save(Box.REPLIES, reply) new_threads.append(reply.id) - q = {"remote_id": create.id} - if not DB.inbox.find_one_and_update( - q, {"$set": {"meta.thread_root_parent": root_reply}} - ): - DB.outbox.update_one(q, {"$set": {"meta.thread_root_parent": root_reply}}) - - DB.threads.update( - {"remote_id": {"$in": new_threads}}, + DB.activities.update_one( + {"remote_id": create.id}, {"$set": {"meta.thread_root_parent": root_reply}} + ) + DB.activities.update( + {"box": Box.REPLIES.value, "remote_id": {"$in": new_threads}}, {"$set": {"meta.thread_root_parent": root_reply}}, ) @@ -423,7 +392,9 @@ def gen_feed(): fg.description(f"{USERNAME} notes") fg.logo(ME.get("icon", {}).get("url")) fg.language("en") - for item in DB.outbox.find({"type": "Create"}, limit=50): + for item in DB.activities.find( + {"box": Box.OUTBOX.value, "type": "Create"}, limit=50 + ): fe = fg.add_entry() fe.id(item["activity"]["object"].get("url")) fe.link(href=item["activity"]["object"].get("url")) @@ -435,7 +406,9 @@ def gen_feed(): def json_feed(path: str) -> Dict[str, Any]: """JSON Feed (https://jsonfeed.org/) document.""" data = [] - for item in DB.outbox.find({"type": "Create"}, limit=50): + for item in DB.activities.find( + {"box": Box.OUTBOX.value, "type": "Create"}, limit=50 + ): data.append( { "id": item["id"], @@ -471,11 +444,15 @@ def build_inbox_json_feed( data = [] cursor = None - q: Dict[str, Any] = {"type": "Create", "meta.deleted": False} + q: Dict[str, Any] = { + "type": "Create", + "meta.deleted": False, + "box": Box.INBOX.value, + } if request_cursor: q["_id"] = {"$lt": request_cursor} - for item in DB.inbox.find(q, limit=50).sort("_id", -1): + for item in DB.activities.find(q, limit=50).sort("_id", -1): actor = ap.get_backend().fetch_iri(item["activity"]["actor"]) data.append( { diff --git a/app.py b/app.py index 3b947dc..2cb4a27 100644 --- a/app.py +++ b/app.py @@ -38,6 +38,7 @@ from werkzeug.utils import secure_filename import activitypub import config +from activitypub import Box from activitypub import embed_collection from config import ADMIN_API_KEY from config import BASE_URL @@ -56,6 +57,7 @@ from config import _drop_db from config import custom_cache_purge_hook from little_boxes import activitypub as ap from little_boxes.activitypub import ActivityType +from little_boxes.activitypub import _to_list from little_boxes.activitypub import clean_activity from little_boxes.activitypub import get_backend from little_boxes.content_helper import parse_markdown @@ -111,18 +113,21 @@ def inject_config(): "activity.object.inReplyTo": None, "meta.deleted": False, } - notes_count = DB.outbox.find( - {"$or": [q, {"type": "Announce", "meta.undo": False}]} + notes_count = DB.activities.find( + {"box": Box.OUTBOX.value, "$or": [q, {"type": "Announce", "meta.undo": False}]} ).count() q = {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} - with_replies_count = DB.outbox.find( - {"$or": [q, {"type": "Announce", "meta.undo": False}]} + with_replies_count = DB.activities.find( + {"box": Box.OUTBOX.value, "$or": [q, {"type": "Announce", "meta.undo": False}]} ).count() - liked_count = DB.outbox.count({ - "meta.deleted": False, - "meta.undo": False, - "type": ActivityType.LIKE.value, - }) + liked_count = DB.activities.count( + { + "box": Box.OUTBOX.value, + "meta.deleted": False, + "meta.undo": False, + "type": ActivityType.LIKE.value, + } + ) return dict( microblogpub_version=VERSION, config=config, @@ -132,6 +137,7 @@ def inject_config(): notes_count=notes_count, liked_count=liked_count, with_replies_count=with_replies_count, + me=ME, ) @@ -172,6 +178,11 @@ def clean_html(html): return bleach.clean(html, tags=ALLOWED_TAGS) +@app.template_filter() +def permalink_id(val): + return str(hash(val)) + + @app.template_filter() def quote_plus(t): return urllib.parse.quote_plus(t) @@ -221,6 +232,13 @@ def format_timeago(val): return val +@app.template_filter() +def has_type(doc, _type): + if _type in _to_list(doc["type"]): + return True + return False + + def _is_img(filename): filename = filename.lower() if ( @@ -370,9 +388,7 @@ def login(): payload = u2f.begin_authentication(ID, devices) session["challenge"] = payload - return render_template( - "login.html", u2f_enabled=u2f_enabled, me=ME, payload=payload - ) + return render_template("login.html", u2f_enabled=u2f_enabled, payload=payload) @app.route("/remote_follow", methods=["GET", "POST"]) @@ -429,6 +445,45 @@ def u2f_register(): # Activity pub routes +@app.route("/migration1_step1") +@login_required +def tmp_migrate(): + for activity in DB.outbox.find(): + activity["box"] = Box.OUTBOX.value + DB.activities.insert_one(activity) + for activity in DB.inbox.find(): + activity["box"] = Box.INBOX.value + DB.activities.insert_one(activity) + for activity in DB.replies.find(): + activity["box"] = Box.REPLIES.value + DB.activities.insert_one(activity) + return "Done" + + +@app.route("/migration1_step2") +@login_required +def tmp_migrate2(): + for activity in DB.activities.find(): + if ( + activity["box"] == Box.OUTBOX.value + and activity["type"] == ActivityType.LIKE.value + ): + like = ap.parse_activity(activity["activity"]) + obj = like.get_object() + DB.activities.update_one( + {"remote_id": like.id}, + {"$set": {"meta.object": obj.to_dict(embed=True)}}, + ) + elif activity["type"] == ActivityType.ANNOUNCE.value: + announce = ap.parse_activity(activity["activity"]) + obj = announce.get_object() + DB.activities.update_one( + {"remote_id": announce.id}, + {"$set": {"meta.object": obj.to_dict(embed=True)}}, + ) + return "Done" + + @app.route("/") def index(): if is_api_request(): @@ -437,89 +492,44 @@ def index(): # FIXME(tsileo): implements pagination, also for the followers/following page limit = 50 q = { - "type": "Create", - "activity.object.type": "Note", + "box": Box.OUTBOX.value, + "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, "activity.object.inReplyTo": None, "meta.deleted": False, + "meta.undo": False, } c = request.args.get("cursor") if c: q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list( - DB.outbox.find( - {"$or": [q, {"type": "Announce", "meta.undo": False}]}, limit=limit - ).sort("_id", -1) - ) + outbox_data = list(DB.activities.find(q, limit=limit).sort("_id", -1)) cursor = None if outbox_data and len(outbox_data) == limit: cursor = str(outbox_data[-1]["_id"]) - for data in outbox_data: - if data["type"] == "Announce": - if data["activity"]["object"].startswith("http"): - data["ref"] = { - "activity": { - "object": OBJECT_SERVICE.get(data["activity"]["object"]), - "id": "NA", - }, - "meta": {}, - } - print(data) - - - return render_template( - "index.html", - me=ME, - notes=DB.inbox.find( - {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} - ).count(), - followers=DB.followers.count(), - following=DB.following.count(), - outbox_data=outbox_data, - cursor=cursor, - ) + return render_template("index.html", outbox_data=outbox_data, cursor=cursor) @app.route("/with_replies") def with_replies(): + # FIXME(tsileo): implements pagination, also for the followers/following page limit = 50 - q = {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} + q = { + "box": Box.OUTBOX.value, + "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, + "meta.deleted": False, + "meta.undo": False, + } c = request.args.get("cursor") if c: q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list( - DB.outbox.find( - {"$or": [q, {"type": "Announce", "meta.undo": False}]}, limit=limit - ).sort("_id", -1) - ) + outbox_data = list(DB.activities.find(q, limit=limit).sort("_id", -1)) cursor = None if outbox_data and len(outbox_data) == limit: cursor = str(outbox_data[-1]["_id"]) - for data in outbox_data: - if data["type"] == "Announce": - print(data) - if data["activity"]["object"].startswith("http"): - data["ref"] = { - "activity": { - "object": OBJECT_SERVICE.get(data["activity"]["object"]) - }, - "meta": {}, - } - - return render_template( - "index.html", - me=ME, - notes=DB.inbox.find( - {"type": "Create", "activity.object.type": "Note", "meta.deleted": False} - ).count(), - followers=DB.followers.count(), - following=DB.following.count(), - outbox_data=outbox_data, - cursor=cursor, - ) + return render_template("index.html", outbox_data=outbox_data, cursor=cursor) def _build_thread(data, include_children=True): @@ -534,12 +544,7 @@ def _build_thread(data, include_children=True): ) # Fetch the root replies, and the children - replies = ( - [data] - + list(DB.inbox.find(query)) - + list(DB.outbox.find(query)) - + list(DB.threads.find(query)) - ) + replies = [data] + list(DB.activities.find(query)) replies = sorted(replies, key=lambda d: d["activity"]["object"]["published"]) # Index all the IDs in order to build a tree idx = {} @@ -580,7 +585,9 @@ def _build_thread(data, include_children=True): @app.route("/note/") def note_by_id(note_id): - data = DB.outbox.find_one({"remote_id": back.activity_url(note_id)}) + data = DB.activities.find_one( + {"box": Box.OUTBOX.value, "remote_id": back.activity_url(note_id)} + ) if not data: abort(404) if data["meta"].get("deleted", False): @@ -588,7 +595,7 @@ def note_by_id(note_id): thread = _build_thread(data) likes = list( - DB.inbox.find( + DB.activities.find( { "meta.undo": False, "type": ActivityType.LIKE.value, @@ -602,7 +609,7 @@ def note_by_id(note_id): likes = [ACTOR_SERVICE.get(doc["activity"]["actor"]) for doc in likes] shares = list( - DB.inbox.find( + DB.activities.find( { "meta.undo": False, "type": ActivityType.ANNOUNCE.value, @@ -616,7 +623,7 @@ def note_by_id(note_id): shares = [ACTOR_SERVICE.get(doc["activity"]["actor"]) for doc in shares] return render_template( - "note.html", likes=likes, shares=shares, me=ME, thread=thread, note=data + "note.html", likes=likes, shares=shares, thread=thread, note=data ) @@ -636,7 +643,10 @@ def nodeinfo(): "protocols": ["activitypub"], "services": {"inbound": [], "outbound": []}, "openRegistrations": False, - "usage": {"users": {"total": 1}, "localPosts": DB.outbox.count()}, + "usage": { + "users": {"total": 1}, + "localPosts": DB.activities.count({"box": Box.OUTBOX.value}), + }, "metadata": { "sourceCode": "https://github.com/tsileo/microblog.pub", "nodeName": f"@{USERNAME}@{DOMAIN}", @@ -734,12 +744,13 @@ def outbox(): # TODO(tsileo): filter the outbox if not authenticated # FIXME(tsileo): filter deleted, add query support for build_ordered_collection q = { + "box": Box.OUTBOX.value, "meta.deleted": False, # 'type': {'$in': [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, } return jsonify( **activitypub.build_ordered_collection( - DB.outbox, + DB.activities, q=q, cursor=request.args.get("cursor"), map_func=lambda doc: activity_from_doc(doc, embed=True), @@ -765,7 +776,9 @@ def outbox(): @app.route("/outbox/") def outbox_detail(item_id): - doc = DB.outbox.find_one({"remote_id": back.activity_url(item_id)}) + doc = DB.activities.find_one( + {"box": Box.OUTBOX.value, "remote_id": back.activity_url(item_id)} + ) if doc["meta"].get("deleted", False): obj = ap.parse_activity(doc["activity"]) resp = jsonify(**obj.get_object().get_tombstone()) @@ -777,8 +790,12 @@ def outbox_detail(item_id): @app.route("/outbox//activity") def outbox_activity(item_id): # TODO(tsileo): handle Tombstone - data = DB.outbox.find_one( - {"remote_id": back.activity_url(item_id), "meta.deleted": False} + data = DB.activities.find_one( + { + "box": Box.OUTBOX.value, + "remote_id": back.activity_url(item_id), + "meta.deleted": False, + } ) if not data: abort(404) @@ -793,8 +810,12 @@ def outbox_activity_replies(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one( - {"remote_id": back.activity_url(item_id), "meta.deleted": False} + data = DB.activities.find_one( + { + "box": Box.OUTBOX.value, + "remote_id": back.activity_url(item_id), + "meta.deleted": False, + } ) if not data: abort(404) @@ -810,7 +831,7 @@ def outbox_activity_replies(item_id): return jsonify( **activitypub.build_ordered_collection( - DB.inbox, + DB.activities, q=q, cursor=request.args.get("cursor"), map_func=lambda doc: doc["activity"]["object"], @@ -825,8 +846,12 @@ def outbox_activity_likes(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one( - {"remote_id": back.activity_url(item_id), "meta.deleted": False} + data = DB.activities.find_one( + { + "box": Box.OUTBOX.value, + "remote_id": back.activity_url(item_id), + "meta.deleted": False, + } ) if not data: abort(404) @@ -845,7 +870,7 @@ def outbox_activity_likes(item_id): return jsonify( **activitypub.build_ordered_collection( - DB.inbox, + DB.activities, q=q, cursor=request.args.get("cursor"), map_func=lambda doc: remove_context(doc["activity"]), @@ -860,8 +885,12 @@ def outbox_activity_shares(item_id): # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) - data = DB.outbox.find_one( - {"remote_id": back.activity_url(item_id), "meta.deleted": False} + data = DB.activities.find_one( + { + "box": Box.OUTBOX.value, + "remote_id": back.activity_url(item_id), + "meta.deleted": False, + } ) if not data: abort(404) @@ -880,7 +909,7 @@ def outbox_activity_shares(item_id): return jsonify( **activitypub.build_ordered_collection( - DB.inbox, + DB.activities, q=q, cursor=request.args.get("cursor"), map_func=lambda doc: remove_context(doc["activity"]), @@ -893,16 +922,21 @@ def outbox_activity_shares(item_id): @app.route("/admin", methods=["GET"]) @login_required def admin(): - q = {"meta.deleted": False, "meta.undo": False, "type": ActivityType.LIKE.value} - col_liked = DB.outbox.count(q) + q = { + "meta.deleted": False, + "meta.undo": False, + "type": ActivityType.LIKE.value, + "box": Box.OUTBOX.value, + } + col_liked = DB.activities.count(q) return render_template( "admin.html", instances=list(DB.instances.find()), - inbox_size=DB.inbox.count(), - outbox_size=DB.outbox.count(), - object_cache_size=DB.objects_cache.count(), - actor_cache_size=DB.actors_cache.count(), + inbox_size=DB.activities.count({"box": Box.INBOX.value}), + outbox_size=DB.activities.count({"box": Box.OUTBOX.value}), + object_cache_size=0, + actor_cache_size=0, col_liked=col_liked, col_followers=DB.followers.count(), col_following=DB.following.count(), @@ -916,11 +950,9 @@ def new(): content = "" thread = [] if request.args.get("reply"): - data = DB.inbox.find_one({"activity.object.id": request.args.get("reply")}) + data = DB.activities.find_one({"activity.object.id": request.args.get("reply")}) if not data: - data = DB.outbox.find_one({"activity.object.id": request.args.get("reply")}) - if not data: - abort(400) + abort(400) reply = ap.parse_activity(data["activity"]) reply_id = reply.id @@ -930,7 +962,7 @@ def new(): domain = urlparse(actor.id).netloc # FIXME(tsileo): if reply of reply, fetch all participants content = f"@{actor.preferredUsername}@{domain} " - thread = _build_thread(data, include_children=False) + thread = _build_thread(data) return render_template("new.html", reply=reply_id, content=content, thread=thread) @@ -940,43 +972,42 @@ def new(): def notifications(): # FIXME(tsileo): implements pagination, also for the followers/following page limit = 50 - q = { - "type": "Create", + # FIXME(tsileo): show unfollow (performed by the current actor) and liked??? + mentions_query = { + "type": ActivityType.CREATE.value, "activity.object.tag.type": "Mention", "activity.object.tag.name": f"@{USERNAME}@{DOMAIN}", "meta.deleted": False, } - # TODO(tsileo): also include replies via regex on Create replyTo - q = { - "$or": [ - q, - {"type": "Follow"}, - {"type": "Accept"}, - {"type": "Undo", "activity.object.type": "Follow"}, - {"type": "Announce", "activity.object": {"$regex": f"^{BASE_URL}"}}, - {"type": "Create", "activity.object.inReplyTo": {"$regex": f"^{BASE_URL}"}}, - ] + replies_query = { + "type": ActivityType.CREATE.value, + "activity.object.inReplyTo": {"$regex": f"^{BASE_URL}"}, + } + announced_query = { + "type": ActivityType.ANNOUNCE.value, + "activity.object": {"$regex": f"^{BASE_URL}"}, + } + new_followers_query = {"type": ActivityType.FOLLOW.value} + followed_query = {"type": ActivityType.ACCEPT.value} + q = { + "box": Box.INBOX.value, + "$or": [ + mentions_query, + announced_query, + replies_query, + new_followers_query, + followed_query, + ], } - print(q) c = request.args.get("cursor") if c: q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list(DB.inbox.find(q, limit=limit).sort("_id", -1)) + outbox_data = list(DB.activities.find(q, limit=limit).sort("_id", -1)) cursor = None if outbox_data and len(outbox_data) == limit: cursor = str(outbox_data[-1]["_id"]) - # TODO(tsileo): fix the annonce handling, copy it from /stream - # for data in outbox_data: - # if data['type'] == 'Announce': - # print(data) - # if data['activity']['object'].startswith('http') and data['activity']['object'] in objcache: - # data['ref'] = {'activity': {'object': objcache[data['activity']['object']]}, 'meta': {}} - # out.append(data) - # else: - # out.append(data) - return render_template("stream.html", inbox_data=outbox_data, cursor=cursor) @@ -1061,8 +1092,11 @@ def api_like(): @api_required def api_undo(): oid = _user_api_arg("id") - doc = DB.outbox.find_one( - {"$or": [{"remote_id": back.activity_url(oid)}, {"remote_id": oid}]} + doc = DB.activities.find_one( + { + "box": Box.OUTBOX.value, + "$or": [{"remote_id": back.activity_url(oid)}, {"remote_id": oid}], + } ) if not doc: raise ActivityNotFoundError(f"cannot found {oid}") @@ -1080,50 +1114,24 @@ def api_undo(): def stream(): # FIXME(tsileo): implements pagination, also for the followers/following page limit = 100 + c = request.args.get("cursor") q = { - "type": "Create", - "activity.object.type": "Note", - "activity.object.inReplyTo": None, + "box": Box.INBOX.value, + "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, "meta.deleted": False, } - c = request.args.get("cursor") if c: q["_id"] = {"$lt": ObjectId(c)} - outbox_data = list( - DB.inbox.find({"$or": [q, {"type": "Announce"}]}, limit=limit).sort( - "activity.published", -1 - ) + inbox_data = list( + # FIXME(tsileo): reshape using meta.cached_object + DB.activities.find(q, limit=limit).sort("_id", -1) ) cursor = None - if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]["_id"]) + if inbox_data and len(inbox_data) == limit: + cursor = str(inbox_data[-1]["_id"]) - out = [] - objcache = {} - cached = list( - DB.objects_cache.find({"meta.part_of_stream": True}, limit=limit * 3).sort( - "meta.announce_published", -1 - ) - ) - for c in cached: - objcache[c["object_id"]] = c["cached_object"] - for data in outbox_data: - if data["type"] == "Announce": - if ( - data["activity"]["object"].startswith("http") - and data["activity"]["object"] in objcache - ): - data["ref"] = { - "activity": {"object": objcache[data["activity"]["object"]]}, - "meta": {}, - } - out.append(data) - else: - print("OMG", data) - else: - out.append(data) - return render_template("stream.html", inbox_data=out, cursor=cursor) + return render_template("stream.html", inbox_data=inbox_data, cursor=cursor) @app.route("/inbox", methods=["GET", "POST"]) @@ -1138,8 +1146,8 @@ def inbox(): return jsonify( **activitypub.build_ordered_collection( - DB.inbox, - q={"meta.deleted": False}, + DB.activities, + q={"meta.deleted": False, "box": Box.INBOX.value}, cursor=request.args.get("cursor"), map_func=lambda doc: remove_context(doc["activity"]), ) @@ -1198,9 +1206,9 @@ def api_debug(): return flask_jsonify(message="DB dropped") return flask_jsonify( - inbox=DB.inbox.count(), - outbox=DB.outbox.count(), - outbox_data=without_id(DB.outbox.find()), + inbox=DB.activities.count({"box": Box.INBOX.value}), + outbox=DB.activities.count({"box": Box.OUTBOX.value}), + outbox_data=without_id(DB.activities.find({"box": Box.OUTBOX.value})), ) @@ -1305,8 +1313,13 @@ def api_stream(): def api_block(): actor = _user_api_arg("actor") - existing = DB.outbox.find_one( - {"type": ActivityType.BLOCK.value, "activity.object": actor, "meta.undo": False} + existing = DB.activities.find_one( + { + "box": Box.OUTBOX.value, + "type": ActivityType.BLOCK.value, + "activity.object": actor, + "meta.undo": False, + } ) if existing: return _user_api_response(activity=existing["activity"]["id"]) @@ -1346,14 +1359,7 @@ def followers(): followers = [ ACTOR_SERVICE.get(doc["remote_actor"]) for doc in DB.followers.find(limit=50) ] - return render_template( - "followers.html", - me=ME, - notes=DB.inbox.find({"object.object.type": "Note"}).count(), - followers=DB.followers.count(), - following=DB.following.count(), - followers_data=followers, - ) + return render_template("followers.html", followers_data=followers) @app.route("/following") @@ -1370,30 +1376,27 @@ def following(): following = [ ACTOR_SERVICE.get(doc["remote_actor"]) for doc in DB.following.find(limit=50) ] - return render_template( - "following.html", - me=ME, - notes=DB.inbox.find({"object.object.type": "Note"}).count(), - followers=DB.followers.count(), - following=DB.following.count(), - following_data=following, - ) + return render_template("following.html", following_data=following) @app.route("/tags/") def tags(tag): - if not DB.outbox.count( - {"activity.object.tag.type": "Hashtag", "activity.object.tag.name": "#" + tag} + if not DB.activities.count( + { + "box": Box.OUTBOX.value, + "activity.object.tag.type": "Hashtag", + "activity.object.tag.name": "#" + tag, + } ): abort(404) if not is_api_request(): return render_template( "tags.html", tag=tag, - outbox_data=DB.outbox.find( + outbox_data=DB.activities.find( { - "type": "Create", - "activity.object.type": "Note", + "box": Box.OUTBOX.value, + "type": ActivityType.CREATE.value, "meta.deleted": False, "activity.object.tag.type": "Hashtag", "activity.object.tag.name": "#" + tag, @@ -1401,6 +1404,7 @@ def tags(tag): ), ) q = { + "box": Box.OUTBOX.value, "meta.deleted": False, "meta.undo": False, "type": ActivityType.CREATE.value, @@ -1409,7 +1413,7 @@ def tags(tag): } return jsonify( **activitypub.build_ordered_collection( - DB.outbox, + DB.activities, q=q, cursor=request.args.get("cursor"), map_func=lambda doc: doc["activity"]["object"]["id"], @@ -1423,9 +1427,9 @@ def liked(): if not is_api_request(): return render_template( "liked.html", - me=ME, - liked=DB.outbox.find( + liked=DB.activities.find( { + "box": Box.OUTBOX.value, "type": ActivityType.LIKE.value, "meta.deleted": False, "meta.undo": False, @@ -1436,7 +1440,7 @@ def liked(): q = {"meta.deleted": False, "meta.undo": False, "type": ActivityType.LIKE.value} return jsonify( **activitypub.build_ordered_collection( - DB.outbox, + DB.activities, q=q, cursor=request.args.get("cursor"), map_func=lambda doc: doc["activity"]["object"], diff --git a/templates/index.html b/templates/index.html index 09ffb94..285e306 100644 --- a/templates/index.html +++ b/templates/index.html @@ -23,16 +23,15 @@
    {% for item in outbox_data %} - {% if item.type == 'Announce' %} + {% if item | has_type('Announce') %} {% set boost_actor = item.activity.actor | get_actor %}

    {{ boost_actor.name }} boosted

    - {% if item.ref %} - {{ utils.display_note(item.ref, ui=False) }} - {% endif %} - - {% elif item.type == 'Create' %} - {{ utils.display_note(item) }} + {% if item.meta.object %} + {{ utils.display_note(item.meta.object, ui=False) }} {% endif %} + {% elif item | has_type('Create') %} + {{ utils.display_note(item.activity.object, meta=item.meta) }} + {% endif %} {% endfor %}
    diff --git a/templates/liked.html b/templates/liked.html new file mode 100644 index 0000000..b90462d --- /dev/null +++ b/templates/liked.html @@ -0,0 +1,18 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block content %} +
    + + +{% include "header.html" %} + +
    + {% for item in liked %} + {% if item.meta.object %} + {{ utils.display_note(item.meta.object) }} + {% endif %} + {% endfor %} +
    + +
    +{% endblock %} diff --git a/templates/stream.html b/templates/stream.html index 9a0cff7..994e3d5 100644 --- a/templates/stream.html +++ b/templates/stream.html @@ -8,26 +8,32 @@
    {% for item in inbox_data %} - {% if item.type == 'Create' %} - {{ utils.display_note(item, ui=True) }} + {% if item | has_type('Create') %} + {{ utils.display_note(item.activity.object, ui=True) }} {% else %} - {% if item.type == 'Announce' %} - + {% if item | has_type('Announce') %} {% set boost_actor = item.activity.actor | get_actor %}

    {{ boost_actor.name or boost_actor.preferredUsername }} boosted

    - {% if item.ref %} - {{ utils.display_note(item.ref, ui=True) }} + {% if item.meta.object %} + {{ utils.display_note(item.meta.object, ui=True) }} {% endif %} {% endif %} - {% if item.type == 'Follow' %} -

    {{ item.activity.actor }} followed you

    - {% elif item.type == 'Accept' %} -

    you followed {{ item.activity.actor }}

    - {% elif item.type == 'Undo' %} -

    {{ item.activity.actor }} unfollowed you

    + {% if item | has_type('Follow') %} +

    new follower +

    + {{ utils.display_actor_inline(item.activity.actor | get_actor, size=50) }} +
    + + {% elif item | has_type('Accept') %} +

    you started following

    +
    + {{ utils.display_actor_inline(item.activity.actor | get_actor, size=50) }} +
    + {% else %} + {% endif %} diff --git a/templates/tags.html b/templates/tags.html index 0df993c..3beaa8b 100644 --- a/templates/tags.html +++ b/templates/tags.html @@ -23,7 +23,7 @@

    #{{ tag }}

    {% for item in outbox_data %} - {{ utils.display_note(item) }} + {{ utils.display_note(item.activity.object, meta=item.meta) }} {% endfor %}
    diff --git a/templates/utils.html b/templates/utils.html index 4314fbd..9fd10c2 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -14,27 +14,10 @@ {%- endmacro %} -{% macro display_actor(follower) -%} - -
    -
    -{% if not follower.icon %} - -{% else %} -{% endif %} -
    -
    -

    {{ follower.name or follower.preferredUsername }}

    -@{{ follower.preferredUsername }}@{{ follower.url | domain }} -
    {{ follower.summary | safe }}
    -
    -
    -     -{%- endmacro %} -{% macro display_note(item, perma=False, ui=False, likes=[], shares=[]) -%} -{% set actor = item.activity.object.attributedTo | get_actor %} -
    +{% macro display_note(obj, perma=False, ui=False, likes=[], shares=[], meta={}) -%} +{% set actor = obj.attributedTo | get_actor %} +
    @@ -47,78 +30,79 @@ {% if not perma %} - - + + {% endif %} - {% if item.activity.object.summary %}

    {{ item.activity.object.summary }}

    {% endif %} + {% if obj.summary %}

    {{ obj.summary | clean }}

    {% endif %}
    - {{ item.activity.object.content | safe }} + {{ obj.content | clean | safe }}
    - {% if item.activity.object.attachment %} + {% if obj.attachment %}
    - {% if item.activity.object.attachment | not_only_imgs %} + {% if obj.attachment | not_only_imgs %}

    Attachment

    {% endif %}
    {% endif %}
    -{% if perma %}{{ item.activity.object.published | format_time }} +{% if perma %}{{ obj.published | format_time }} {% else %} -permalink +permalink -{% if item.meta.count_reply %}{{ item.meta.count_reply }} replies{% endif %} -{% if item.meta.count_boost %}{{ item.meta.count_boost }} boosts{% endif %} -{% if item.meta.count_like %}{{ item.meta.count_like }} likes{% endif %} +{% if meta.count_reply %}{{ meta.count_reply }} replies{% endif %} +{% if meta.count_boost %}{{ meta.count_boost }} boosts{% endif %} +{% if meta.count_like %}{{ meta.count_like }} likes{% endif %} {% endif %} {% if ui and session.logged_in %} -{% set aid = item.activity.object.id | quote_plus %} +{% set aid = obj.id | quote_plus %} reply -{% set redir = request.path + "#activity-" + item['_id'].__str__() %} +{% set perma_id = obj.id | permalink_id %} +{% set redir = request.path + "#activity-" + perma_id %} -{% if item.meta.boosted %} +{% if meta.boosted %}
    - +
    {% else %}
    - +
    {% endif %} -{% if item.meta.liked %} +{% if meta.liked %}
    - +
    {% else %}
    - +
    @@ -127,17 +111,17 @@ {% endif %} {% if session.logged_in %} -{% if item.activity.id | is_from_outbox %} +{% if obj.id | is_from_outbox %}
    - +
    {% else %}
    - +
    @@ -151,14 +135,14 @@
    {% if likes %}
    -

    {{ item.meta.count_like }} likes

    {% for like in likes %} +

    {{ meta.count_like }} likes

    {% for like in likes %} {{ display_actor_inline(like) }} {% endfor %}
    {% endif %} {% if shares %}
    -

    {{ item.meta.count_boost }} boosts

    {% for boost in shares %} +

    {{ meta.count_boost }} boosts

    {% for boost in shares %} {{ display_actor_inline(boost) }} {% endfor %}
    @@ -177,9 +161,9 @@ {% macro display_thread(thread, likes=[], shares=[]) -%} {% for reply in thread %} {% if reply._requested %} -{{ display_note(reply, perma=True, ui=False, likes=likes, shares=shares) }} +{{ display_note(reply.activity.object, perma=True, ui=False, likes=likes, shares=shares, meta=reply.meta) }} {% else %} -{{ display_note(reply, perma=False, ui=True) }} +{{ display_note(reply.activity.object, perma=False, ui=True, meta=reply.meta) }} {% endif %} {% endfor %} {% endmacro -%} From 68695fed396a939989d8459be18ef2d378a63e79 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 29 Jun 2018 22:42:53 +0200 Subject: [PATCH 0150/1425] Allow br tags for note content --- app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app.py b/app.py index 2cb4a27..84ef9d1 100644 --- a/app.py +++ b/app.py @@ -153,6 +153,7 @@ ALLOWED_TAGS = [ "abbr", "acronym", "b", + "br", "blockquote", "code", "pre", From 340d9155991d9a45be327ef8ff7d85c7cac8c793 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 30 Jun 2018 11:12:03 +0200 Subject: [PATCH 0151/1425] Fix the migration handler --- app.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/app.py b/app.py index 84ef9d1..71ad305 100644 --- a/app.py +++ b/app.py @@ -465,23 +465,26 @@ def tmp_migrate(): @login_required def tmp_migrate2(): for activity in DB.activities.find(): - if ( - activity["box"] == Box.OUTBOX.value - and activity["type"] == ActivityType.LIKE.value - ): - like = ap.parse_activity(activity["activity"]) - obj = like.get_object() - DB.activities.update_one( - {"remote_id": like.id}, - {"$set": {"meta.object": obj.to_dict(embed=True)}}, - ) - elif activity["type"] == ActivityType.ANNOUNCE.value: - announce = ap.parse_activity(activity["activity"]) - obj = announce.get_object() - DB.activities.update_one( - {"remote_id": announce.id}, - {"$set": {"meta.object": obj.to_dict(embed=True)}}, - ) + try: + if ( + activity["box"] == Box.OUTBOX.value + and activity["type"] == ActivityType.LIKE.value + ): + like = ap.parse_activity(activity["activity"]) + obj = like.get_object() + DB.activities.update_one( + {"remote_id": like.id}, + {"$set": {"meta.object": obj.to_dict(embed=True)}}, + ) + elif activity["type"] == ActivityType.ANNOUNCE.value: + announce = ap.parse_activity(activity["activity"]) + obj = announce.get_object() + DB.activities.update_one( + {"remote_id": announce.id}, + {"$set": {"meta.object": obj.to_dict(embed=True)}}, + ) + except Exception: + pass return "Done" From 4e7fb005fa27dc45009eb3420c43a3d23962a985 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Jul 2018 11:04:57 +0200 Subject: [PATCH 0152/1425] Increase gunicorn timeout (to help with the migration) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 10bf4ca..8a7a764 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,4 +3,4 @@ ADD . /app WORKDIR /app RUN pip install -r requirements.txt ENV FLASK_APP=app.py -CMD ["gunicorn", "-w", "2", "-b", "0.0.0.0:5005", "--log-level", "debug", "app:app"] +CMD ["gunicorn", "-t", "300", "-w", "2", "-b", "0.0.0.0:5005", "--log-level", "debug", "app:app"] From 3bf77b2e82d1367b25a1c663f1b979b489c221c2 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Jul 2018 11:05:33 +0200 Subject: [PATCH 0153/1425] Fix the migration --- app.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/app.py b/app.py index 71ad305..750b66a 100644 --- a/app.py +++ b/app.py @@ -464,27 +464,27 @@ def tmp_migrate(): @app.route("/migration1_step2") @login_required def tmp_migrate2(): + # Remove buggy OStatus announce + DB.activities.remove({"activity.object": {"$regex": f"^tag:"}, "type": ActivityType.ANNOUNCE.value}) + # Cache the object for activity in DB.activities.find(): - try: - if ( - activity["box"] == Box.OUTBOX.value - and activity["type"] == ActivityType.LIKE.value - ): - like = ap.parse_activity(activity["activity"]) - obj = like.get_object() - DB.activities.update_one( - {"remote_id": like.id}, - {"$set": {"meta.object": obj.to_dict(embed=True)}}, - ) - elif activity["type"] == ActivityType.ANNOUNCE.value: - announce = ap.parse_activity(activity["activity"]) - obj = announce.get_object() - DB.activities.update_one( - {"remote_id": announce.id}, - {"$set": {"meta.object": obj.to_dict(embed=True)}}, - ) - except Exception: - pass + if ( + activity["box"] == Box.OUTBOX.value + and activity["type"] == ActivityType.LIKE.value + ): + like = ap.parse_activity(activity["activity"]) + obj = like.get_object() + DB.activities.update_one( + {"remote_id": like.id}, + {"$set": {"meta.object": obj.to_dict(embed=True)}}, + ) + elif activity["type"] == ActivityType.ANNOUNCE.value: + announce = ap.parse_activity(activity["activity"]) + obj = announce.get_object() + DB.activities.update_one( + {"remote_id": announce.id}, + {"$set": {"meta.object": obj.to_dict(embed=True)}}, + ) return "Done" From a5b7e21600adcf0bc75b4d6e549ce06ef245f4f4 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Jul 2018 11:40:44 +0200 Subject: [PATCH 0154/1425] Better theme support --- config.py | 33 ++++++++- sass/base_theme.scss | 21 ------ sass/dark.scss | 8 +++ sass/light.scss | 9 +++ static/base.css | 160 ------------------------------------------ static/css/.gitignore | 2 - static/css/theme.css | 1 - templates/layout.html | 4 +- 8 files changed, 51 insertions(+), 187 deletions(-) create mode 100644 sass/dark.scss create mode 100644 sass/light.scss delete mode 100644 static/base.css delete mode 100644 static/css/.gitignore delete mode 100644 static/css/theme.css diff --git a/config.py b/config.py index 1a00bb7..3e883c0 100644 --- a/config.py +++ b/config.py @@ -1,9 +1,11 @@ import os import subprocess from datetime import datetime +from enum import Enum import requests import yaml +import sass from itsdangerous import JSONWebSignatureSerializer from pymongo import MongoClient @@ -13,6 +15,19 @@ from utils.key import get_key from utils.key import get_secret_key +class ThemeStyle(Enum): + LIGHT = "light" + DARK = "dark" + + +DEFAULT_THEME_STYLE = ThemeStyle.LIGHT.value + +DEFAULT_THEME_PRIMARY_COLOR = { + ThemeStyle.LIGHT: "#1d781d", # Green + ThemeStyle.DARK: "#e14eea", # Purple +} + + def noop(): pass @@ -54,8 +69,22 @@ with open(os.path.join(KEY_DIR, "me.yml")) as f: ICON_URL = conf["icon_url"] PASS = conf["pass"] PUBLIC_INSTANCES = conf.get("public_instances", []) - # TODO(tsileo): choose dark/light style - THEME_COLOR = conf.get("theme_color") + + # Theme-related config + theme_conf = conf.get("theme", {}) + THEME_STYLE = ThemeStyle(theme_conf.get("style", DEFAULT_THEME_STYLE)) + THEME_COLOR = theme_conf.get("color", DEFAULT_THEME_PRIMARY_COLOR[THEME_STYLE]) + + +SASS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "sass") +theme_css = f"$primary-color: {THEME_COLOR};\n" +with open(os.path.join(SASS_DIR, f"{THEME_STYLE.value}.scss")) as f: + theme_css += f.read() + theme_css += '\n' +with open(os.path.join(SASS_DIR, "base_theme.scss")) as f: + raw_css = theme_css + f.read() + CSS = sass.compile(string=raw_css) + USER_AGENT = ( f"{requests.utils.default_user_agent()} (microblog.pub/{VERSION}; +{BASE_URL})" diff --git a/sass/base_theme.scss b/sass/base_theme.scss index 1cbe524..c54114d 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -1,24 +1,3 @@ -// a4.io dark theme -$background-color: #060606; -$background-light: #222; -$color: #808080; -$color-title-link: #fefefe; -$color-summary: #ddd; -$color-light: #bbb; -$color-menu-background: #222; -$color-note-link: #666; -$primary-color: #f7ca18; - -$background-color: #eee; -$background-light: #ccc; -$color: #111; -$color-title-link: #333; -$color-light: #555; -$color-summary: #111; -$color-note-link: #333; -$color-menu-background: #ddd; -$primary-color: #1d781d; - .note-container p:first-child { margin-top: 0; } diff --git a/sass/dark.scss b/sass/dark.scss new file mode 100644 index 0000000..68a9f40 --- /dev/null +++ b/sass/dark.scss @@ -0,0 +1,8 @@ +$background-color: #060606; +$background-light: #222; +$color: #808080; +$color-title-link: #fefefe; +$color-summary: #ddd; +$color-light: #bbb; +$color-menu-background: #222; +$color-note-link: #666; diff --git a/sass/light.scss b/sass/light.scss new file mode 100644 index 0000000..9c4c251 --- /dev/null +++ b/sass/light.scss @@ -0,0 +1,9 @@ +$background-color: #eee; +$background-light: #ccc; +$color: #111; +$color-title-link: #333; +$color-light: #555; +$color-summary: #111; +$color-note-link: #333; +$color-menu-background: #ddd; +// $primary-color: #1d781d; diff --git a/static/base.css b/static/base.css deleted file mode 100644 index aeab1f3..0000000 --- a/static/base.css +++ /dev/null @@ -1,160 +0,0 @@ -.note-container p:first-child { - margin-top: 0; -} -html, body { - height: 100%; -} -body { - background-color: #060606; - color: #808080; -display: flex; -flex-direction: column; -} -.base-container { - flex: 1 0 auto; -} -.footer { -flex-shrink: 0; -} -a, h1, h2, h3, h4, h5, h6 { - color: #fefefe; -} -a { - text-decoration: none; -} -a:hover { - text-decoration: underline; -} -.gold { - color: #f7ca18; -} - -#header { - margin-bottom: 40px; -} -#header .h-card { -} -#header .title { -font-size:1.2em;padding-right:15px;color:#fefefe; -} -#header .title:hover { -text-decoration: none; -} -#header .menu { -padding: 20px 0 10px 0; -} -#header .menu ul { -display:inline;list-style-type:none;padding:0; -} -#header .menu li { -float:left; -padding-right:10px; -margin-bottom:10px; -} -#header .menu a { - padding: 2px 7px; -} -#header .menu a.selected { -background:#f7ca18;color:#634806; -border-radius:2px; -} -#header .menu a:hover { -background:#f7ca18;color:#060606; -text-decoration: none; -} -#container { -width:90%;max-width: 720px;margin:40px auto; -} -#container #notes { -margin-top:20px; -} -.actor-box { -display:block;text-decoration:none;margin-bottom:40px; -} -.actor-box .actor-icon { -width: 100%; -max-width:120px; -border-radius:2px; -} -.actor-box h3 { margin:0; } -.note .l { -color:#666; -} -.note { -display:flex;margin-bottom:70px; -} -.note .h-card { -flex:initial;width:50px; -} -.note .u-photo { -width:50px;border-radius:2px; -} -.note .note-wrapper { -flex:1;padding-left:15px; -} -.note .bottom-bar { -margin-top:10px; -} -.bar-item { -background: #222; -padding: 5px; -color:#bbb; -margin-right:5px; -border-radius:2px; -} -.bottom-bar .perma-item { -margin-right:5px; -} -.bottom-bar a.bar-item:hover { - text-decoration: none; -} -.note .img-attachment { -max-width:100%; -border-radius:2px; -} -.note h3 { -font-size:1.1em;color:#ccc; -} -.note .note-container { -clear:right;padding:10px 0; -} -.note strong { -font-weight:600; -} -.footer > div { -width:90%;max-width: 720px;margin:40px auto; -} -.footer a, .footer a:hover, .footer a:visited { - text-decoration:underline; - color:#808080; -} -.summary { -color: #ddd; -font-size:1.3em; -margin-top:50px; -margin-bottom:70px; -} -.summary a, .summay a:hover { -color:#ddd; -text-decoration:underline; -} -#followers, #following, #new { - margin-top:50px; -} -#admin { - margin-top:50px; -} -textarea, input { - background: #222; - padding: 10px; - color: #bbb; - border: 0px; - border-radius: 2px; -} -input { - padding: 10px; -} -input[type=submit] { - color: #f7ca18; - text-transform: uppercase; -} diff --git a/static/css/.gitignore b/static/css/.gitignore deleted file mode 100644 index d6b7ef3..0000000 --- a/static/css/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/static/css/theme.css b/static/css/theme.css deleted file mode 100644 index 138747f..0000000 --- a/static/css/theme.css +++ /dev/null @@ -1 +0,0 @@ -.note-container p:first-child{margin-top:0}html,body{height:100%}body{background-color:#eee;color:#111;display:flex;flex-direction:column}.base-container{flex:1 0 auto}.footer{flex-shrink:0}a,h1,h2,h3,h4,h5,h6{color:#333}a{text-decoration:none}a:hover{text-decoration:underline}.gold{color:#1d781d}#header{margin-bottom:70px}#header .title{font-size:1.2em;padding-right:15px;color:#333}#header .title:hover{text-decoration:none}#header .subtitle-username{color:#111}#header .menu{clear:both;padding:0 0 10px 0}#header .menu ul{display:inline;list-style-type:none;padding:0}#header .menu ul li{float:left;padding-right:10px;margin-bottom:10px}#header .menu a{padding:5px 10px}#header .menu a small.badge{background-color:#ddd;color:#555;border-radius:2px;margin-left:5px;padding:3px 5px 0px 5px;font-weight:bold}#header .menu a.selected{background:#1d781d;color:#eee;border-radius:2px}#header .menu a.selected .badge{color:#1d781d;background:#eee}#header .menu a:hover{background:#1d781d;color:#eee;text-decoration:none;border-radius:2px}#header .menu a:hover .badge{color:#1d781d;background:#eee}#container{width:90%;max-width:720px;margin:40px auto}#container #notes{margin-top:20px}.actor-box{display:block;text-decoration:none;margin-bottom:40px}.actor-box .actor-icon{width:120px;border-radius:2px}.actor-box h3{margin:0}.actor-box .actor-inline{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.actor-box-big{display:block;text-decoration:none}.actor-box-big .actor-box-wrapper{margin-bottom:40px}.actor-box-big .actor-box-wrapper .actor-icon{width:120px;border-radius:2px}.actor-box-big .actor-box-wrapper h3{margin:0}.note{display:flex;margin-bottom:70px}.note .l{color:#333}.note .h-card{flex:initial;width:50px}.note .u-photo{width:50px;border-radius:2px}.note .note-wrapper{flex:1;padding-left:15px}.note .bottom-bar{margin-top:10px}.note .img-attachment{max-width:100%;border-radius:2px}.note h3{font-size:1.1em;color:#555}.note strong{font-weight:600}.note .note-container{clear:right;padding:10px 0}.bar-item{background:#ddd;padding:5px;color:#555;margin-right:5px;border-radius:2px}button.bar-item{border:0}form.action-form{display:inline}.perma{font-size:1.25em}.bottom-bar .perma-item{margin-right:5px}.bottom-bar a.bar-item:hover{text-decoration:none}.footer>div{width:90%;max-width:720px;margin:40px auto}.footer a,.footer a:hover,.footer a:visited{text-decoration:underline;color:#111}.summary{color:#111;font-size:1.1em;margin-top:10px;margin-bottom:30px}.summary a,.summay a:hover{color:#111;text-decoration:underline}#followers,#following,#new{margin-top:50px}#admin{margin-top:50px}textarea,input{background:#ddd;padding:10px;color:#555;border:0px;border-radius:2px}input{padding:10px}input[type=submit]{color:#1d781d;text-transform:uppercase} diff --git a/templates/layout.html b/templates/layout.html index b6f159d..6d4190f 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -6,7 +6,6 @@ {% block title %}{{ config.NAME }}{% endblock %} - microblog.pub - @@ -15,6 +14,9 @@ {% if config.THEME_COLOR %}{% endif %} +
    From 8449ff0d6ee2c819c53e4c4f41eab8d19e3088a2 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Jul 2018 12:48:40 +0200 Subject: [PATCH 0155/1425] Switch to python:3 for the Docker base image In order to have the Python 3.7 upgrade automatically --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8a7a764..d5ac544 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6 +FROM python:3.7 ADD . /app WORKDIR /app RUN pip install -r requirements.txt From aea8a80fe122ed62123f64e047865cc6f6a7089f Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Jul 2018 12:49:16 +0200 Subject: [PATCH 0156/1425] More design tweaks --- config.py | 2 +- sass/base_theme.scss | 29 +++++++++++++++++++++++++++++ templates/header.html | 3 +-- templates/index.html | 4 ++-- templates/layout.html | 2 +- templates/stream.html | 6 +++--- templates/utils.html | 6 ++++-- 7 files changed, 41 insertions(+), 11 deletions(-) diff --git a/config.py b/config.py index 3e883c0..a5a6bb7 100644 --- a/config.py +++ b/config.py @@ -24,7 +24,7 @@ DEFAULT_THEME_STYLE = ThemeStyle.LIGHT.value DEFAULT_THEME_PRIMARY_COLOR = { ThemeStyle.LIGHT: "#1d781d", # Green - ThemeStyle.DARK: "#e14eea", # Purple + ThemeStyle.DARK: "#33ff00", # Purple } diff --git a/sass/base_theme.scss b/sass/base_theme.scss index c54114d..ec5bd98 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -28,6 +28,23 @@ a:hover { .gold { color: $primary-color; } +.pcolor { + color: $primary-color; +} + +.remote-follow-button { + background: $color-menu-background; + color: $color-light; + text-decoration: none; + padding: 5px 8px; + margin-top: 5px; + border-radius: 2px; +} +.remote-follow-button:hover { + text-decoration: none; + background: $primary-color; + color: $background-color; +} #header { margin-bottom: 70px; @@ -171,6 +188,14 @@ a:hover { } } +.bar-item-no-hover { + background: $color-menu-background; + padding: 5px; + color: $color-light; + margin-right:5px; + border-radius:2px; +} + .bar-item { background: $color-menu-background; padding: 5px; @@ -178,6 +203,10 @@ a:hover { margin-right:5px; border-radius:2px; } +.bar-item:hover { + background: $primary-color; + color: $background-color; +} button.bar-item { border: 0 } diff --git a/templates/header.html b/templates/header.html index 167cc1d..0bfe9a4 100644 --- a/templates/header.html +++ b/templates/header.html @@ -4,8 +4,7 @@ {{ config.NAME }} @{{ config.USERNAME }}@{{ config.DOMAIN }} - Remote follow - + Remote follow
    {{ config.SUMMARY | safe }} diff --git a/templates/index.html b/templates/index.html index 285e306..79fcc96 100644 --- a/templates/index.html +++ b/templates/index.html @@ -25,12 +25,12 @@ {% if item | has_type('Announce') %} {% set boost_actor = item.activity.actor | get_actor %} -

    {{ boost_actor.name }} boosted

    +

    {{ boost_actor.name }} boosted

    {% if item.meta.object %} {{ utils.display_note(item.meta.object, ui=False) }} {% endif %} {% elif item | has_type('Create') %} - {{ utils.display_note(item.activity.object, meta=item.meta) }} + {{ utils.display_note(item.activity.object, meta=item.meta, no_color=True) }} {% endif %} {% endfor %} diff --git a/templates/layout.html b/templates/layout.html index 6d4190f..4c74129 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -15,7 +15,7 @@ {% if config.THEME_COLOR %}{% endif %} diff --git a/templates/stream.html b/templates/stream.html index 994e3d5..d900d3f 100644 --- a/templates/stream.html +++ b/templates/stream.html @@ -14,20 +14,20 @@ {% if item | has_type('Announce') %} {% set boost_actor = item.activity.actor | get_actor %} -

    {{ boost_actor.name or boost_actor.preferredUsername }} boosted

    +

    {{ boost_actor.name or boost_actor.preferredUsername }} boosted

    {% if item.meta.object %} {{ utils.display_note(item.meta.object, ui=True) }} {% endif %} {% endif %} {% if item | has_type('Follow') %} -

    new follower +

    new follower

    {{ utils.display_actor_inline(item.activity.actor | get_actor, size=50) }}
    {% elif item | has_type('Accept') %} -

    you started following

    +

    you started following

    {{ utils.display_actor_inline(item.activity.actor | get_actor, size=50) }}
    diff --git a/templates/utils.html b/templates/utils.html index 9fd10c2..a0f2bec 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -15,7 +15,7 @@ -{% macro display_note(obj, perma=False, ui=False, likes=[], shares=[], meta={}) -%} +{% macro display_note(obj, perma=False, ui=False, likes=[], shares=[], meta={}, no_color=False) -%} {% set actor = obj.attributedTo | get_actor %}
    @@ -26,7 +26,9 @@
    - {{ actor.name or actor.preferredUsername }} @{{ actor.preferredUsername }}@{{ actor.url | domain }} + {{ actor.name or actor.preferredUsername }} + @{% if not no_color and obj.id | is_from_outbox %}{{ actor.preferredUsername }}{% else %}{{ actor.preferredUsername }}{% endif %}@ + {% if not no_color and obj.id | is_from_outbox %}{{ actor.url | domain }}{% else %}{{ actor.url | domain }}{% endif %} {% if not perma %} From cd6f8727c043b721047e3c6b56689ed6f374780e Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Jul 2018 12:49:40 +0200 Subject: [PATCH 0157/1425] fix formatting --- app.py | 4 +++- config.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index 750b66a..63ca676 100644 --- a/app.py +++ b/app.py @@ -465,7 +465,9 @@ def tmp_migrate(): @login_required def tmp_migrate2(): # Remove buggy OStatus announce - DB.activities.remove({"activity.object": {"$regex": f"^tag:"}, "type": ActivityType.ANNOUNCE.value}) + DB.activities.remove( + {"activity.object": {"$regex": f"^tag:"}, "type": ActivityType.ANNOUNCE.value} + ) # Cache the object for activity in DB.activities.find(): if ( diff --git a/config.py b/config.py index a5a6bb7..ec73d0c 100644 --- a/config.py +++ b/config.py @@ -4,8 +4,8 @@ from datetime import datetime from enum import Enum import requests -import yaml import sass +import yaml from itsdangerous import JSONWebSignatureSerializer from pymongo import MongoClient @@ -80,7 +80,7 @@ SASS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "sass") theme_css = f"$primary-color: {THEME_COLOR};\n" with open(os.path.join(SASS_DIR, f"{THEME_STYLE.value}.scss")) as f: theme_css += f.read() - theme_css += '\n' + theme_css += "\n" with open(os.path.join(SASS_DIR, "base_theme.scss")) as f: raw_css = theme_css + f.read() CSS = sass.compile(string=raw_css) From d376e53d2f1e7150254aed5b69452bed2a32ef39 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Jul 2018 12:51:26 +0200 Subject: [PATCH 0158/1425] Fix Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d5ac544..8202116 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7 +FROM python:3 ADD . /app WORKDIR /app RUN pip install -r requirements.txt From 9332b348eb27df2dbcdf59595ec252f5d28bb426 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Jul 2018 21:32:12 +0200 Subject: [PATCH 0159/1425] Bugfix template --- app.py | 5 +++++ templates/stream.html | 10 ++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 63ca676..aced9f0 100644 --- a/app.py +++ b/app.py @@ -994,6 +994,10 @@ def notifications(): "activity.object": {"$regex": f"^{BASE_URL}"}, } new_followers_query = {"type": ActivityType.FOLLOW.value} + unfollow_query = { + "type": ActivityType.UNDO.value, + "activity.object.type": ActivityType.FOLLOW.value, + } followed_query = {"type": ActivityType.ACCEPT.value} q = { "box": Box.INBOX.value, @@ -1003,6 +1007,7 @@ def notifications(): replies_query, new_followers_query, followed_query, + unfollow_query, ], } c = request.args.get("cursor") diff --git a/templates/stream.html b/templates/stream.html index d900d3f..f5c42a6 100644 --- a/templates/stream.html +++ b/templates/stream.html @@ -21,13 +21,19 @@ {% endif %} {% if item | has_type('Follow') %} -

    new follower +

    new follower

    {{ utils.display_actor_inline(item.activity.actor | get_actor, size=50) }}
    {% elif item | has_type('Accept') %} -

    you started following

    +

    you started following

    +
    + {{ utils.display_actor_inline(item.activity.actor | get_actor, size=50) }} +
    + + {% elif item | has_type('Undo') %} +

    unfollowed you

    {{ utils.display_actor_inline(item.activity.actor | get_actor, size=50) }}
    From 0c7c3ccae369642013a4f6221170fba68f8b3d01 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Jul 2018 23:25:26 +0200 Subject: [PATCH 0160/1425] Tweak the admin menu --- sass/base_theme.scss | 26 ++++++++++++++++++++++++-- templates/header.html | 7 ------- templates/layout.html | 12 ++++++++++++ 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/sass/base_theme.scss b/sass/base_theme.scss index ec5bd98..e4e2a37 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -45,7 +45,30 @@ a:hover { background: $primary-color; color: $background-color; } - +#admin-menu { + list-style-type: none; + display: inline; + padding: 10px; + width: 720px; + margin:0 auto; + background: $color-menu-background; + color: $color-light; + border-radius-bottom-left: 2px; + border-radius-bottom-right: 2px; + .left { float: left; } + .right { float: right; } + li { + a { text-decoration: none; } + .admin-title { + text-transform: uppercase; + font-weight: bold; + } + padding-right:10px; + .selected, a:hover { + color: $primary-color; + } + } +} #header { margin-bottom: 70px; @@ -69,7 +92,6 @@ a:hover { padding: 0; li { float:left; - padding-right:10px; margin-bottom:10px; } } diff --git a/templates/header.html b/templates/header.html index 0bfe9a4..1eb54a8 100644 --- a/templates/header.html +++ b/templates/header.html @@ -19,13 +19,6 @@
  • Following {{ following_count }}
    • -{% if logged_in %} -
  • /stream
    • -
  • /notifs
    • -
  • /new
    • -
  • /admin
    • -
  • /logout
    • -{% endif %}
    diff --git a/templates/layout.html b/templates/layout.html index 4c74129..cece912 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -19,6 +19,18 @@ +{% if logged_in %} + +{% endif %} + +
    {% block content %}{% endblock %}
    From 9c664ad29dc1ae7cbfb10d072f8e973884f382a3 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Jul 2018 23:28:15 +0200 Subject: [PATCH 0161/1425] Fix css issue --- templates/utils.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/templates/utils.html b/templates/utils.html index a0f2bec..ba0f51b 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -27,8 +27,7 @@
    {{ actor.name or actor.preferredUsername }} - @{% if not no_color and obj.id | is_from_outbox %}{{ actor.preferredUsername }}{% else %}{{ actor.preferredUsername }}{% endif %}@ - {% if not no_color and obj.id | is_from_outbox %}{{ actor.url | domain }}{% else %}{{ actor.url | domain }}{% endif %} + @{% if not no_color and obj.id | is_from_outbox %}{{ actor.preferredUsername }}{% else %}{{ actor.preferredUsername }}{% endif %}@{% if not no_color and obj.id | is_from_outbox %}{{ actor.url | domain }}{% else %}{{ actor.url | domain }}{% endif %} {% if not perma %} From c6ae9793d5672bb92767e26991aaec487791b2d8 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Jul 2018 23:46:58 +0200 Subject: [PATCH 0162/1425] Allow to attach files from the admin UI when posting a note --- app.py | 70 ++++++++++++++++------------------------------ templates/new.html | 3 +- 2 files changed, 26 insertions(+), 47 deletions(-) diff --git a/app.py b/app.py index aced9f0..a5eeacd 100644 --- a/app.py +++ b/app.py @@ -1223,51 +1223,6 @@ def api_debug(): ) -@app.route("/api/upload", methods=["POST"]) -@api_required -def api_upload(): - file = request.files["file"] - rfilename = secure_filename(file.filename) - prefix = hashlib.sha256(os.urandom(32)).hexdigest()[:6] - mtype = mimetypes.guess_type(rfilename)[0] - filename = f"{prefix}_{rfilename}" - file.save(os.path.join("static", "media", filename)) - - # Remove EXIF metadata - if filename.lower().endswith(".jpg") or filename.lower().endswith(".jpeg"): - piexif.remove(os.path.join("static", "media", filename)) - - print("upload OK") - print(filename) - attachment = [ - { - "mediaType": mtype, - "name": rfilename, - "type": "Document", - "url": BASE_URL + f"/static/media/{filename}", - } - ] - print(attachment) - content = request.args.get("content") - to = request.args.get("to") - note = ap.Note( - attributedTo=MY_PERSON.id, - cc=[ID + "/followers"], - to=[to if to else ap.AS_PUBLIC], - content=content, # TODO(tsileo): handle markdown - attachment=attachment, - ) - print("post_note_init") - print(note) - create = note.build_create() - print(create) - print(create.to_dict()) - OUTBOX.post(create) - print("posted") - - return Response(status=201, response="OK") - - @app.route("/api/new_note", methods=["POST"]) @api_required def api_new_note(): @@ -1293,7 +1248,7 @@ def api_new_note(): if tag["type"] == "Mention": cc.append(tag["href"]) - note = ap.Note( + raw_note = dict( attributedTo=MY_PERSON.id, cc=list(set(cc)), to=[to if to else ap.AS_PUBLIC], @@ -1302,6 +1257,29 @@ def api_new_note(): source={"mediaType": "text/markdown", "content": source}, inReplyTo=reply.id if reply else None, ) + + if 'file' in request.files: + file = request.files["file"] + rfilename = secure_filename(file.filename) + prefix = hashlib.sha256(os.urandom(32)).hexdigest()[:6] + mtype = mimetypes.guess_type(rfilename)[0] + filename = f"{prefix}_{rfilename}" + file.save(os.path.join("static", "media", filename)) + + # Remove EXIF metadata + if filename.lower().endswith(".jpg") or filename.lower().endswith(".jpeg"): + piexif.remove(os.path.join("static", "media", filename)) + + raw_note['attachment'] = [ + { + "mediaType": mtype, + "name": rfilename, + "type": "Document", + "url": BASE_URL + f"/static/media/{filename}", + } + ] + + note = ap.Note(**raw_note) create = note.build_create() OUTBOX.post(create) diff --git a/templates/new.html b/templates/new.html index bf658aa..a14bec7 100644 --- a/templates/new.html +++ b/templates/new.html @@ -11,11 +11,12 @@ {% else %}

    New note

    {% endif %} -
    + {% if reply %}{% endif %} +
    From 1ef476b9b983b615c0e6901244c86b10875d6d90 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 3 Jul 2018 08:25:40 +0200 Subject: [PATCH 0163/1425] Fix follower template --- templates/utils.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/utils.html b/templates/utils.html index ba0f51b..ab61ca5 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -1,5 +1,5 @@ {% macro display_actor_inline(follower, size=50) -%} - + {% if not follower.icon %} @@ -8,7 +8,7 @@
    {{ follower.name or follower.preferredUsername }}
    -@{{ follower.preferredUsername }}@{{ follower.url | domain }} +@{{ follower.preferredUsername }}@{{ follower.get_url() | domain }}
    {%- endmacro %} @@ -20,14 +20,14 @@
    - +
    - {{ actor.name or actor.preferredUsername }} - @{% if not no_color and obj.id | is_from_outbox %}{{ actor.preferredUsername }}{% else %}{{ actor.preferredUsername }}{% endif %}@{% if not no_color and obj.id | is_from_outbox %}{{ actor.url | domain }}{% else %}{{ actor.url | domain }}{% endif %} + {{ actor.name or actor.preferredUsername }} + @{% if not no_color and obj.id | is_from_outbox %}{{ actor.preferredUsername }}{% else %}{{ actor.preferredUsername }}{% endif %}@{% if not no_color and obj.id | is_from_outbox %}{{ actor.get_url() | domain }}{% else %}{{ actor.get_url() | domain }}{% endif %} {% if not perma %} From d31e2f8d1d03bb027f40128aceb1a5af3659516c Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 3 Jul 2018 23:29:55 +0200 Subject: [PATCH 0164/1425] Add support for Link object --- app.py | 24 ++++++++++++++++++++++-- templates/utils.html | 10 +++++----- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/app.py b/app.py index a5eeacd..f2ec403 100644 --- a/app.py +++ b/app.py @@ -209,6 +209,14 @@ def domain(url): return urlparse(url).netloc +@app.template_filter() +def get_url(u): + if isinstance(u, dict): + return u["href"] + else: + return u + + @app.template_filter() def get_actor(url): if not url: @@ -674,6 +682,11 @@ def wellknown_nodeinfo(): ) +# @app.route('/fake_feed') +# def fake_feed(): +# return 'https://lol3.tun.a4.io/fake_feedhttps://lol3.tun.a4.io/fake' + + @app.route("/.well-known/webfinger") def wellknown_webfinger(): """Enable WebFinger support, required for Mastodon interopability.""" @@ -695,6 +708,13 @@ def wellknown_webfinger(): "rel": "http://ostatus.org/schema/1.0/subscribe", "template": BASE_URL + "/authorize_follow?profile={uri}", }, + # {"rel": "magic-public-key", "href": KEY.to_magic_key()}, + # {"rel": "salmon", "href": BASE_URL + "/salmon"}, + # { + # "rel": "http://schemas.google.com/g/2010#updates-from", + # "type": "application/atom+xml", + # "href": f"{BASE_URL}/fake_feed", + # }, ], } @@ -1258,7 +1278,7 @@ def api_new_note(): inReplyTo=reply.id if reply else None, ) - if 'file' in request.files: + if "file" in request.files: file = request.files["file"] rfilename = secure_filename(file.filename) prefix = hashlib.sha256(os.urandom(32)).hexdigest()[:6] @@ -1270,7 +1290,7 @@ def api_new_note(): if filename.lower().endswith(".jpg") or filename.lower().endswith(".jpeg"): piexif.remove(os.path.join("static", "media", filename)) - raw_note['attachment'] = [ + raw_note["attachment"] = [ { "mediaType": mtype, "name": rfilename, diff --git a/templates/utils.html b/templates/utils.html index ab61ca5..070fc9f 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -1,5 +1,5 @@ {% macro display_actor_inline(follower, size=50) -%} - + {% if not follower.icon %} @@ -8,7 +8,7 @@
    {{ follower.name or follower.preferredUsername }}
    -@{{ follower.preferredUsername }}@{{ follower.get_url() | domain }} +@{{ follower.preferredUsername }}@{{ follower.url | get_url | domain }}
    {%- endmacro %} @@ -20,14 +20,14 @@
    - +
    - {{ actor.name or actor.preferredUsername }} - @{% if not no_color and obj.id | is_from_outbox %}{{ actor.preferredUsername }}{% else %}{{ actor.preferredUsername }}{% endif %}@{% if not no_color and obj.id | is_from_outbox %}{{ actor.get_url() | domain }}{% else %}{{ actor.get_url() | domain }}{% endif %} + {{ actor.name or actor.preferredUsername }} + @{% if not no_color and obj.id | is_from_outbox %}{{ actor.preferredUsername }}{% else %}{{ actor.preferredUsername }}{% endif %}@{% if not no_color and obj.id | is_from_outbox %}{{ actor.url | get_url | domain }}{% else %}{{ actor.url | get_url | domain }}{% endif %} {% if not perma %} From f73ec6b96aabb7db40d65008fe35918a4e85320b Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 3 Jul 2018 23:40:13 +0200 Subject: [PATCH 0165/1425] More template fixes --- templates/layout.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/layout.html b/templates/layout.html index cece912..c457ff6 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -9,7 +9,8 @@ - + + From 5331874c77a7d0c9895209520b578ca83f479dca Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 3 Jul 2018 23:43:34 +0200 Subject: [PATCH 0166/1425] Tweak the CSS --- sass/base_theme.scss | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sass/base_theme.scss b/sass/base_theme.scss index e4e2a37..41c0432 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -91,8 +91,9 @@ a:hover { list-style-type: none; padding: 0; li { - float:left; - margin-bottom:10px; + float: left; + margin-bottom: 10px; + margin-right: 10px; } } a { @@ -131,7 +132,7 @@ a:hover { #container { width: 90%; max-width: 720px; - margin: 40px auto; + margin: 30px auto; } #container #notes { margin-top: 20px; From 7eb52884b8aaa35aad899262b08679526eb73dc5 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 3 Jul 2018 23:54:16 +0200 Subject: [PATCH 0167/1425] Compress the CSS --- config.py | 2 +- templates/layout.html | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/config.py b/config.py index ec73d0c..38cf802 100644 --- a/config.py +++ b/config.py @@ -83,7 +83,7 @@ with open(os.path.join(SASS_DIR, f"{THEME_STYLE.value}.scss")) as f: theme_css += "\n" with open(os.path.join(SASS_DIR, "base_theme.scss")) as f: raw_css = theme_css + f.read() - CSS = sass.compile(string=raw_css) + CSS = sass.compile(string=raw_css, output_style='compressed') USER_AGENT = ( diff --git a/templates/layout.html b/templates/layout.html index c457ff6..d9bdfd0 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -15,9 +15,7 @@ {% if config.THEME_COLOR %}{% endif %} - + {% if logged_in %} From 0b8e0896c1c0f4ea0daa90982d3efe2133c0f134 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 4 Jul 2018 00:40:23 +0200 Subject: [PATCH 0168/1425] Tweak the webfinger response --- app.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app.py b/app.py index f2ec403..f768ad5 100644 --- a/app.py +++ b/app.py @@ -690,6 +690,7 @@ def wellknown_nodeinfo(): @app.route("/.well-known/webfinger") def wellknown_webfinger(): """Enable WebFinger support, required for Mastodon interopability.""" + # TODO(tsileo): move this to little-boxes? resource = request.args.get("resource") if resource not in [f"acct:{USERNAME}@{DOMAIN}", ID]: abort(404) @@ -708,13 +709,12 @@ def wellknown_webfinger(): "rel": "http://ostatus.org/schema/1.0/subscribe", "template": BASE_URL + "/authorize_follow?profile={uri}", }, - # {"rel": "magic-public-key", "href": KEY.to_magic_key()}, - # {"rel": "salmon", "href": BASE_URL + "/salmon"}, - # { - # "rel": "http://schemas.google.com/g/2010#updates-from", - # "type": "application/atom+xml", - # "href": f"{BASE_URL}/fake_feed", - # }, + {"rel": "magic-public-key", "href": KEY.to_magic_key()}, + { + "href": BASE_URL, + "rel": "http://webfinger.net/rel/profile-page", + "type": "text/html", + }, ], } From 8556cd29efe315deab1cc7d6db9c34e095c192bc Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 4 Jul 2018 21:08:45 +0200 Subject: [PATCH 0169/1425] Add avatar to the webfinger response --- app.py | 6 ++++++ config.py | 9 +++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index f768ad5..ed8ca9c 100644 --- a/app.py +++ b/app.py @@ -46,6 +46,7 @@ from config import DB from config import DEBUG_MODE from config import DOMAIN from config import HEADERS +from config import ICON_URL from config import ID from config import JWT from config import KEY @@ -715,6 +716,11 @@ def wellknown_webfinger(): "rel": "http://webfinger.net/rel/profile-page", "type": "text/html", }, + { + "href": ICON_URL, + "rel": "http://webfinger.net/rel/avatar", + "type": mimetypes.guess_type(ICON_URL)[0], + }, ], } diff --git a/config.py b/config.py index 38cf802..5f2da9c 100644 --- a/config.py +++ b/config.py @@ -1,3 +1,4 @@ +import mimetypes import os import subprocess from datetime import datetime @@ -83,7 +84,7 @@ with open(os.path.join(SASS_DIR, f"{THEME_STYLE.value}.scss")) as f: theme_css += "\n" with open(os.path.join(SASS_DIR, "base_theme.scss")) as f: raw_css = theme_css + f.read() - CSS = sass.compile(string=raw_css, output_style='compressed') + CSS = sass.compile(string=raw_css, output_style="compressed") USER_AGENT = ( @@ -136,6 +137,10 @@ ME = { "summary": SUMMARY, "endpoints": {}, "url": ID, - "icon": {"mediaType": "image/png", "type": "Image", "url": ICON_URL}, + "icon": { + "mediaType": mimetypes.guess_type(ICON_URL)[0], + "type": "Image", + "url": ICON_URL, + }, "publicKey": KEY.to_dict(), } From 13c63e473aa32cf5640b6a647ef1db3b5f6a2f7d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 5 Jul 2018 01:02:51 +0200 Subject: [PATCH 0170/1425] Start to cache actor icon --- app.py | 44 +++++++++++++++++++++++++++++++++++++ config.py | 1 + requirements.txt | 1 + templates/utils.html | 2 +- utils/img.py | 52 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 utils/img.py diff --git a/app.py b/app.py index ed8ca9c..1ffa156 100644 --- a/app.py +++ b/app.py @@ -10,6 +10,7 @@ from datetime import timezone from functools import wraps from typing import Any from typing import Dict +from typing import Tuple from urllib.parse import urlencode from urllib.parse import urlparse @@ -45,6 +46,7 @@ from config import BASE_URL from config import DB from config import DEBUG_MODE from config import DOMAIN +from config import GRIDFS from config import HEADERS from config import ICON_URL from config import ID @@ -69,9 +71,12 @@ from little_boxes.httpsig import HTTPSigAuth from little_boxes.httpsig import verify_request from little_boxes.webfinger import get_actor_url from little_boxes.webfinger import get_remote_follow_template +from utils.img import ImageCache from utils.key import get_secret_key from utils.object_service import ObjectService +IMAGE_CACHE = ImageCache(GRIDFS) + OBJECT_SERVICE = ACTOR_SERVICE = ObjectService() back = activitypub.MicroblogPubBackend() @@ -180,6 +185,30 @@ def clean_html(html): return bleach.clean(html, tags=ALLOWED_TAGS) +_GRIDFS_CACHE: Dict[Tuple[str, int], str] = {} + + +def _get_actor_icon_url(url, size): + k = (url, size) + cached = _GRIDFS_CACHE.get(k) + if cached: + return cached + + doc = IMAGE_CACHE.fs.find_one({"url": url, "size": size}) + if doc: + u = f"/img/{str(doc._id)}" + _GRIDFS_CACHE[k] = u + return u + + IMAGE_CACHE.cache_actor_icon(url) + return _get_actor_icon_url(url, size) + + +@app.template_filter() +def get_actor_icon_url(url, size): + return _get_actor_icon_url(url, size) + + @app.template_filter() def permalink_id(val): return str(hash(val)) @@ -357,6 +386,21 @@ def handle_activitypub_error(error): # App routes + +@app.route("/img/") +def serve_img(img_id): + f = IMAGE_CACHE.fs.get(ObjectId(img_id)) + resp = app.response_class(f, direct_passthrough=True, mimetype=f.content_type) + resp.headers.set("Content-Length", f.length) + resp.headers.set("ETag", f.md5) + resp.headers.set( + "Last-Modified", f.uploadDate.strftime("%a, %d %b %Y %H:%M:%S GMT") + ) + resp.headers.set("Cache-Control", "public,max-age=31536000,immutable") + resp.headers.set("Content-Encoding", "gzip") + return resp + + ####### # Login diff --git a/config.py b/config.py index 5f2da9c..906a4b9 100644 --- a/config.py +++ b/config.py @@ -97,6 +97,7 @@ mongo_client = MongoClient( DB_NAME = "{}_{}".format(USERNAME, DOMAIN.replace(".", "_")) DB = mongo_client[DB_NAME] +GRIDFS = mongo_client[f"{DB_NAME}_gridfs"] def _drop_db(): diff --git a/requirements.txt b/requirements.txt index 3c3b9a3..21ce34c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,3 +19,4 @@ passlib git+https://github.com/erikriver/opengraph.git git+https://github.com/tsileo/little-boxes.git pyyaml +pillow diff --git a/templates/utils.html b/templates/utils.html index 070fc9f..e456ddb 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -20,7 +20,7 @@
    - +
    diff --git a/utils/img.py b/utils/img.py new file mode 100644 index 0000000..2b08e58 --- /dev/null +++ b/utils/img.py @@ -0,0 +1,52 @@ +import base64 +from gzip import GzipFile +from io import BytesIO +from typing import Any + +import gridfs +import requests +from PIL import Image + + +def load(url): + """Initializes a `PIL.Image` from the URL.""" + # TODO(tsileo): user agent + resp = requests.get(url, stream=True) + resp.raise_for_status() + try: + image = Image.open(BytesIO(resp.raw.read())) + finally: + resp.close() + return image + + +def to_data_uri(img): + out = BytesIO() + img.save(out, format=img.format) + out.seek(0) + data = base64.b64encode(out.read()).decode("utf-8") + return f"data:{img.get_format_mimetype()};base64,{data}" + + +class ImageCache(object): + def __init__(self, gridfs_db: str) -> None: + self.fs = gridfs.GridFS(gridfs_db) + + def cache_actor_icon(self, url: str): + if self.fs.find_one({"url": url}): + return + i = load(url) + for size in [50, 80]: + t1 = i.copy() + t1.thumbnail((size, size)) + with BytesIO() as buf: + f1 = GzipFile(mode='wb', fileobj=buf) + t1.save(f1, format=i.format) + f1.close() + buf.seek(0) + self.fs.put( + buf, url=url, size=size, content_type=i.get_format_mimetype() + ) + + def get_file(self, url: str, size: int) -> Any: + return self.fs.find_one({"url": url, "size": size}) From 6d3ca7fcc5b16b51b5947919e319c33692198e4e Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 5 Jul 2018 01:08:39 +0200 Subject: [PATCH 0171/1425] More image caching --- templates/utils.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/utils.html b/templates/utils.html index e456ddb..a00935a 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -4,7 +4,7 @@ {% if not follower.icon %} {% else %} -{% endif %} +{% endif %}
    {{ follower.name or follower.preferredUsername }}
    From a6c1ede629cd7a6c0b85da63036cb8959f5cc19b Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 5 Jul 2018 01:20:13 +0200 Subject: [PATCH 0172/1425] Fix CI --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a523ee1..8540f59 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ python: services: - docker install: - - sudo apt-get install -y curl + - sudo apt-get install -y curl python-tk - sudo pip install -U pip - sudo curl -L https://github.com/docker/compose/releases/download/1.21.2/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose From 5bb7875bc9177aaf643bdaba634014b8ea815b54 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 5 Jul 2018 01:29:21 +0200 Subject: [PATCH 0173/1425] Pyyaml is broken with Python3.7, back to 3.6 for now --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8202116..8a7a764 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3 +FROM python:3.6 ADD . /app WORKDIR /app RUN pip install -r requirements.txt From a2a64a54fd22c93f7077b2ce3242079feba27097 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 5 Jul 2018 01:45:56 +0200 Subject: [PATCH 0174/1425] Remove duplicate webfinger entry --- app.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/app.py b/app.py index 1ffa156..16ed99b 100644 --- a/app.py +++ b/app.py @@ -755,11 +755,6 @@ def wellknown_webfinger(): "template": BASE_URL + "/authorize_follow?profile={uri}", }, {"rel": "magic-public-key", "href": KEY.to_magic_key()}, - { - "href": BASE_URL, - "rel": "http://webfinger.net/rel/profile-page", - "type": "text/html", - }, { "href": ICON_URL, "rel": "http://webfinger.net/rel/avatar", From e8ee900c600480d4d8fcfcb8796e2796c83bd08d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 5 Jul 2018 22:27:29 +0200 Subject: [PATCH 0175/1425] Cache attachments and actor icons Fixes #17 --- app.py | 42 ++++++++++++--- templates/layout.html | 2 +- templates/utils.html | 4 +- utils/img.py | 115 +++++++++++++++++++++++++++++++++++------- 4 files changed, 134 insertions(+), 29 deletions(-) diff --git a/app.py b/app.py index 16ed99b..b35e05d 100644 --- a/app.py +++ b/app.py @@ -37,10 +37,12 @@ from passlib.hash import bcrypt from u2flib_server import u2f from werkzeug.utils import secure_filename + import activitypub import config from activitypub import Box from activitypub import embed_collection +from config import USER_AGENT from config import ADMIN_API_KEY from config import BASE_URL from config import DB @@ -72,10 +74,12 @@ from little_boxes.httpsig import verify_request from little_boxes.webfinger import get_actor_url from little_boxes.webfinger import get_remote_follow_template from utils.img import ImageCache +from utils.img import Kind from utils.key import get_secret_key from utils.object_service import ObjectService +from typing import Optional -IMAGE_CACHE = ImageCache(GRIDFS) +IMAGE_CACHE = ImageCache(GRIDFS, USER_AGENT) OBJECT_SERVICE = ACTOR_SERVICE = ObjectService() @@ -185,28 +189,33 @@ def clean_html(html): return bleach.clean(html, tags=ALLOWED_TAGS) -_GRIDFS_CACHE: Dict[Tuple[str, int], str] = {} +_GRIDFS_CACHE: Dict[Tuple[Kind, str, Optional[int]], str] = {} -def _get_actor_icon_url(url, size): - k = (url, size) +def _get_file_url(url, size, kind): + k = (kind, url, size) cached = _GRIDFS_CACHE.get(k) if cached: return cached - doc = IMAGE_CACHE.fs.find_one({"url": url, "size": size}) + doc = IMAGE_CACHE.get_file(url, size, kind) if doc: u = f"/img/{str(doc._id)}" _GRIDFS_CACHE[k] = u return u - IMAGE_CACHE.cache_actor_icon(url) - return _get_actor_icon_url(url, size) + IMAGE_CACHE.cache(url, kind) + return _get_file_url(url, size, kind) @app.template_filter() def get_actor_icon_url(url, size): - return _get_actor_icon_url(url, size) + return _get_file_url(url, size, Kind.ACTOR_ICON) + + +@app.template_filter() +def get_attachment_url(url, size): + return _get_file_url(url, size, Kind.ATTACHMENT) @app.template_filter() @@ -543,6 +552,23 @@ def tmp_migrate2(): return "Done" +@app.route("/migration2") +@login_required +def tmp_migrate3(): + for activity in DB.activities.find(): + try: + activity = ap.parse_activity(activity["activity"]) + actor = activity.get_actor() + if actor.icon: + IMAGE_CACHE.cache(actor.icon["url"], Kind.ACTOR_ICON) + if activity.type == ActivityType.CREATE.value: + for attachment in activity.get_object()._data.get("attachment", []): + IMAGE_CACHE.cache(attachment["url"], Kind.ATTACHMENT) + except: + app.logger.exception('failed') + return "Done" + + @app.route("/") def index(): if is_api_request(): diff --git a/templates/layout.html b/templates/layout.html index d9bdfd0..b6c8d3e 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -4,7 +4,7 @@ -{% block title %}{{ config.NAME }}{% endblock %} - microblog.pub +{% block title %}{{ config.NAME }}{% endblock %}'s microblog diff --git a/templates/utils.html b/templates/utils.html index a00935a..0b158c3 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -48,9 +48,9 @@ {% endif %} {% for a in obj.attachment %} {% if a.url | is_img %} - + {% else %} -
  • {% if a.filename %}{{ a.filename }}{% else %}{{ a.url }}{% endif %}
    • +
  • {% if a.filename %}{{ a.filename }}{% else %}{{ a.url }}{% endif %}
    • {% endif %} {% endfor %} {% if obj.attachment | not_only_imgs %} diff --git a/utils/img.py b/utils/img.py index 2b08e58..dfe43d6 100644 --- a/utils/img.py +++ b/utils/img.py @@ -2,22 +2,20 @@ import base64 from gzip import GzipFile from io import BytesIO from typing import Any +import mimetypes +from enum import Enum import gridfs import requests from PIL import Image -def load(url): +def load(url, user_agent): """Initializes a `PIL.Image` from the URL.""" # TODO(tsileo): user agent - resp = requests.get(url, stream=True) - resp.raise_for_status() - try: - image = Image.open(BytesIO(resp.raw.read())) - finally: - resp.close() - return image + with requests.get(url, stream=True, headers={"User-Agent": user_agent}) as resp: + resp.raise_for_status() + return Image.open(BytesIO(resp.raw.read())) def to_data_uri(img): @@ -28,25 +26,106 @@ def to_data_uri(img): return f"data:{img.get_format_mimetype()};base64,{data}" -class ImageCache(object): - def __init__(self, gridfs_db: str) -> None: - self.fs = gridfs.GridFS(gridfs_db) +class Kind(Enum): + ATTACHMENT = "attachment" + ACTOR_ICON = "actor_icon" - def cache_actor_icon(self, url: str): - if self.fs.find_one({"url": url}): + +class ImageCache(object): + def __init__(self, gridfs_db: str, user_agent: str) -> None: + self.fs = gridfs.GridFS(gridfs_db) + self.user_agent = user_agent + + def cache_attachment(self, url: str) -> None: + if self.fs.find_one({"url": url, "kind": Kind.ATTACHMENT.value}): return - i = load(url) + if ( + url.endswith(".png") + or url.endswith(".jpg") + or url.endswith(".jpeg") + or url.endswith(".gif") + ): + i = load(url, self.user_agent) + # Save the original attachment (gzipped) + with BytesIO() as buf: + f1 = GzipFile(mode="wb", fileobj=buf) + i.save(f1, format=i.format) + f1.close() + buf.seek(0) + self.fs.put( + buf, + url=url, + size=None, + content_type=i.get_format_mimetype(), + kind=Kind.ATTACHMENT.value, + ) + # Save a thumbnail (gzipped) + i.thumbnail((720, 720)) + with BytesIO() as buf: + f1 = GzipFile(mode="wb", fileobj=buf) + i.save(f1, format=i.format) + f1.close() + buf.seek(0) + self.fs.put( + buf, + url=url, + size=720, + content_type=i.get_format_mimetype(), + kind=Kind.ATTACHMENT.value, + ) + return + + # The attachment is not an image, download and save it anyway + with requests.get( + url, stream=True, headers={"User-Agent": self.user_agent} + ) as resp: + resp.raise_for_status() + with BytesIO() as buf: + f1 = GzipFile(mode="wb", fileobj=buf) + for chunk in resp.iter_content(): + if chunk: + f1.write(chunk) + f1.close() + buf.seek(0) + self.fs.put( + buf, + url=url, + size=None, + content_type=mimetypes.guess_type(url)[0], + kind=Kind.ATTACHMENT.value, + ) + + def cache_actor_icon(self, url: str) -> None: + if self.fs.find_one({"url": url, "kind": Kind.ACTOR_ICON.value}): + return + i = load(url, self.user_agent) for size in [50, 80]: t1 = i.copy() t1.thumbnail((size, size)) with BytesIO() as buf: - f1 = GzipFile(mode='wb', fileobj=buf) + f1 = GzipFile(mode="wb", fileobj=buf) t1.save(f1, format=i.format) f1.close() buf.seek(0) self.fs.put( - buf, url=url, size=size, content_type=i.get_format_mimetype() + buf, + url=url, + size=size, + content_type=i.get_format_mimetype(), + kind=Kind.ACTOR_ICON.value, ) - def get_file(self, url: str, size: int) -> Any: - return self.fs.find_one({"url": url, "size": size}) + def cache(self, url: str, kind: Kind) -> None: + if kind == Kind.ACTOR_ICON: + self.cache_actor_icon(url) + else: + self.cache_attachment(url) + + def get_actor_icon(self, url: str, size: int) -> Any: + return self._get_file(url, size, Kind.ACTOR_ICON) + + def get_attachment(self, url: str, size: int) -> Any: + return self._get_file(url, size, Kind.ATTACHMENT) + + def get_file(self, url: str, size: int, kind: Kind) -> Any: + return self.fs.find_one({"url": url, "size": size, "kind": kind.value}) From 0d7a1b9b5a848d42723ad2c2bd6dd23ba49b1c5c Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 5 Jul 2018 22:42:38 +0200 Subject: [PATCH 0176/1425] Bugfix --- utils/img.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/img.py b/utils/img.py index dfe43d6..35d1049 100644 --- a/utils/img.py +++ b/utils/img.py @@ -122,10 +122,10 @@ class ImageCache(object): self.cache_attachment(url) def get_actor_icon(self, url: str, size: int) -> Any: - return self._get_file(url, size, Kind.ACTOR_ICON) + return self.get_file(url, size, Kind.ACTOR_ICON) def get_attachment(self, url: str, size: int) -> Any: - return self._get_file(url, size, Kind.ATTACHMENT) + return self.get_file(url, size, Kind.ATTACHMENT) def get_file(self, url: str, size: int, kind: Kind) -> Any: return self.fs.find_one({"url": url, "size": size, "kind": kind.value}) From acafc1cc85b4c2a970d96ee1c9192b3db02832cc Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 6 Jul 2018 23:15:49 +0200 Subject: [PATCH 0177/1425] Store uplaods in MongoDB too, start pagination --- app.py | 112 ++++++++++++++++++++++++------------- config.py | 2 + sass/base_theme.scss | 10 +++- templates/index.html | 13 +++++ templates/layout.html | 3 +- utils/{img.py => media.py} | 53 +++++++++++++----- 6 files changed, 138 insertions(+), 55 deletions(-) rename utils/{img.py => media.py} (73%) diff --git a/app.py b/app.py index b35e05d..18dd98e 100644 --- a/app.py +++ b/app.py @@ -1,5 +1,4 @@ import binascii -import hashlib import json import logging import mimetypes @@ -8,15 +7,16 @@ import urllib from datetime import datetime from datetime import timezone from functools import wraps +from io import BytesIO from typing import Any from typing import Dict +from typing import Optional from typing import Tuple from urllib.parse import urlencode from urllib.parse import urlparse import bleach import mf2py -import piexif import pymongo import timeago from bson.objectid import ObjectId @@ -37,24 +37,22 @@ from passlib.hash import bcrypt from u2flib_server import u2f from werkzeug.utils import secure_filename - import activitypub import config from activitypub import Box from activitypub import embed_collection -from config import USER_AGENT from config import ADMIN_API_KEY from config import BASE_URL from config import DB from config import DEBUG_MODE from config import DOMAIN -from config import GRIDFS from config import HEADERS from config import ICON_URL from config import ID from config import JWT from config import KEY from config import ME +from config import MEDIA_CACHE from config import PASS from config import USERNAME from config import VERSION @@ -73,13 +71,9 @@ from little_boxes.httpsig import HTTPSigAuth from little_boxes.httpsig import verify_request from little_boxes.webfinger import get_actor_url from little_boxes.webfinger import get_remote_follow_template -from utils.img import ImageCache -from utils.img import Kind from utils.key import get_secret_key +from utils.media import Kind from utils.object_service import ObjectService -from typing import Optional - -IMAGE_CACHE = ImageCache(GRIDFS, USER_AGENT) OBJECT_SERVICE = ACTOR_SERVICE = ObjectService() @@ -198,13 +192,13 @@ def _get_file_url(url, size, kind): if cached: return cached - doc = IMAGE_CACHE.get_file(url, size, kind) + doc = MEDIA_CACHE.get_file(url, size, kind) if doc: - u = f"/img/{str(doc._id)}" + u = f"/media/{str(doc._id)}" _GRIDFS_CACHE[k] = u return u - IMAGE_CACHE.cache(url, kind) + MEDIA_CACHE.cache(url, kind) return _get_file_url(url, size, kind) @@ -395,10 +389,35 @@ def handle_activitypub_error(error): # App routes +ROBOTS_TXT = """User-agent: * +Disallow: /admin/ +Disallow: /static/ +Disallow: /media/ +Disallow: /uploads/""" -@app.route("/img/") -def serve_img(img_id): - f = IMAGE_CACHE.fs.get(ObjectId(img_id)) + +@app.route("/robots.txt") +def robots_txt(): + return Response(response=ROBOTS_TXT, headers={"Content-Type": "text/plain"}) + + +@app.route("/media/") +def serve_media(media_id): + f = MEDIA_CACHE.fs.get(ObjectId(media_id)) + resp = app.response_class(f, direct_passthrough=True, mimetype=f.content_type) + resp.headers.set("Content-Length", f.length) + resp.headers.set("ETag", f.md5) + resp.headers.set( + "Last-Modified", f.uploadDate.strftime("%a, %d %b %Y %H:%M:%S GMT") + ) + resp.headers.set("Cache-Control", "public,max-age=31536000,immutable") + resp.headers.set("Content-Encoding", "gzip") + return resp + + +@app.route("/uploads//") +def serve_uploads(oid, fname): + f = MEDIA_CACHE.fs.get(ObjectId(oid)) resp = app.response_class(f, direct_passthrough=True, mimetype=f.content_type) resp.headers.set("Content-Length", f.length) resp.headers.set("ETag", f.md5) @@ -560,12 +579,12 @@ def tmp_migrate3(): activity = ap.parse_activity(activity["activity"]) actor = activity.get_actor() if actor.icon: - IMAGE_CACHE.cache(actor.icon["url"], Kind.ACTOR_ICON) + MEDIA_CACHE.cache(actor.icon["url"], Kind.ACTOR_ICON) if activity.type == ActivityType.CREATE.value: for attachment in activity.get_object()._data.get("attachment", []): - IMAGE_CACHE.cache(attachment["url"], Kind.ATTACHMENT) - except: - app.logger.exception('failed') + MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT) + except Exception: + app.logger.exception("failed") return "Done" @@ -574,8 +593,10 @@ def index(): if is_api_request(): return jsonify(**ME) - # FIXME(tsileo): implements pagination, also for the followers/following page - limit = 50 + older_than = newer_than = None + query_sort = -1 + first_page = not request.args.get('older_than') and not request.args.get('newer_than') + limit = 5 q = { "box": Box.OUTBOX.value, "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, @@ -583,16 +604,35 @@ def index(): "meta.deleted": False, "meta.undo": False, } - c = request.args.get("cursor") - if c: - q["_id"] = {"$lt": ObjectId(c)} + query_older_than = request.args.get("older_than") + query_newer_than = request.args.get("newer_than") + if query_older_than: + q["_id"] = {"$lt": ObjectId(query_older_than)} + elif query_newer_than: + q["_id"] = {"$gt": ObjectId(query_newer_than)} + query_sort = 1 - outbox_data = list(DB.activities.find(q, limit=limit).sort("_id", -1)) - cursor = None - if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]["_id"]) + outbox_data = list(DB.activities.find(q, limit=limit+1).sort("_id", query_sort)) + outbox_len = len(outbox_data) + outbox_data = sorted(outbox_data[:limit], key=lambda x: str(x["_id"]), reverse=True) - return render_template("index.html", outbox_data=outbox_data, cursor=cursor) + if query_older_than: + newer_than = str(outbox_data[0]["_id"]) + if outbox_len == limit + 1: + older_than = str(outbox_data[-1]["_id"]) + elif query_newer_than: + older_than = str(outbox_data[-1]["_id"]) + if outbox_len == limit + 1: + newer_than = str(outbox_data[0]["_id"]) + elif first_page and outbox_len == limit + 1: + older_than = str(outbox_data[-1]["_id"]) + + return render_template( + "index.html", + outbox_data=outbox_data, + older_than=older_than, + newer_than=newer_than, + ) @app.route("/with_replies") @@ -1352,21 +1392,17 @@ def api_new_note(): if "file" in request.files: file = request.files["file"] rfilename = secure_filename(file.filename) - prefix = hashlib.sha256(os.urandom(32)).hexdigest()[:6] + with BytesIO() as buf: + file.save(buf) + oid = MEDIA_CACHE.save_upload(buf, rfilename) mtype = mimetypes.guess_type(rfilename)[0] - filename = f"{prefix}_{rfilename}" - file.save(os.path.join("static", "media", filename)) - - # Remove EXIF metadata - if filename.lower().endswith(".jpg") or filename.lower().endswith(".jpeg"): - piexif.remove(os.path.join("static", "media", filename)) raw_note["attachment"] = [ { "mediaType": mtype, "name": rfilename, "type": "Document", - "url": BASE_URL + f"/static/media/{filename}", + "url": f"{BASE_URL}/uploads/{oid}/{rfilename}", } ] diff --git a/config.py b/config.py index 906a4b9..c145932 100644 --- a/config.py +++ b/config.py @@ -14,6 +14,7 @@ from little_boxes import strtobool from utils.key import KEY_DIR from utils.key import get_key from utils.key import get_secret_key +from utils.media import MediaCache class ThemeStyle(Enum): @@ -98,6 +99,7 @@ mongo_client = MongoClient( DB_NAME = "{}_{}".format(USERNAME, DOMAIN.replace(".", "_")) DB = mongo_client[DB_NAME] GRIDFS = mongo_client[f"{DB_NAME}_gridfs"] +MEDIA_CACHE = MediaCache(GRIDFS, USER_AGENT) def _drop_db(): diff --git a/sass/base_theme.scss b/sass/base_theme.scss index 41c0432..b5dd5c6 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -31,7 +31,15 @@ a:hover { .pcolor { color: $primary-color; } - +.lcolor { + color: $color-light; +} +.older-link, .newer-linker, .older-link:hover, .newer-link:hover { + text-decoration: none; + padding: 3px; +} +.newer-link { float: right } +.clear { clear: both; } .remote-follow-button { background: $color-menu-background; color: $color-light; diff --git a/templates/index.html b/templates/index.html index 79fcc96..5c2d178 100644 --- a/templates/index.html +++ b/templates/index.html @@ -34,7 +34,20 @@ {% endif %} {% endfor %} +
    + {% if older_than %} + + {% endif %} + {% if newer_than %} + + {% endif %} +
    {% endblock %} +{% block links %} +{% if older_than %}{% endif %} +{% if newer_than %}{% endif %} +{% endblock %} + diff --git a/templates/layout.html b/templates/layout.html index b6c8d3e..da804a3 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -9,11 +9,12 @@ - +{% if not request.args.get("older_than") and not request.args.get("previous_than") %}{% endif %} +{% block links %}{% endblock %} {% if config.THEME_COLOR %}{% endif %} diff --git a/utils/img.py b/utils/media.py similarity index 73% rename from utils/img.py rename to utils/media.py index 35d1049..13e2b0a 100644 --- a/utils/img.py +++ b/utils/media.py @@ -1,20 +1,21 @@ import base64 +import mimetypes +from enum import Enum from gzip import GzipFile from io import BytesIO from typing import Any -import mimetypes -from enum import Enum import gridfs +import piexif import requests from PIL import Image def load(url, user_agent): """Initializes a `PIL.Image` from the URL.""" - # TODO(tsileo): user agent with requests.get(url, stream=True, headers={"User-Agent": user_agent}) as resp: resp.raise_for_status() + resp.raw.decode_content = True return Image.open(BytesIO(resp.raw.read())) @@ -29,9 +30,10 @@ def to_data_uri(img): class Kind(Enum): ATTACHMENT = "attachment" ACTOR_ICON = "actor_icon" + UPLOAD = "upload" -class ImageCache(object): +class MediaCache(object): def __init__(self, gridfs_db: str, user_agent: str) -> None: self.fs = gridfs.GridFS(gridfs_db) self.user_agent = user_agent @@ -62,9 +64,8 @@ class ImageCache(object): # Save a thumbnail (gzipped) i.thumbnail((720, 720)) with BytesIO() as buf: - f1 = GzipFile(mode="wb", fileobj=buf) - i.save(f1, format=i.format) - f1.close() + with GzipFile(mode="wb", fileobj=buf) as f1: + i.save(f1, format=i.format) buf.seek(0) self.fs.put( buf, @@ -81,11 +82,10 @@ class ImageCache(object): ) as resp: resp.raise_for_status() with BytesIO() as buf: - f1 = GzipFile(mode="wb", fileobj=buf) - for chunk in resp.iter_content(): - if chunk: - f1.write(chunk) - f1.close() + with GzipFile(mode="wb", fileobj=buf) as f1: + for chunk in resp.iter_content(): + if chunk: + f1.write(chunk) buf.seek(0) self.fs.put( buf, @@ -103,9 +103,8 @@ class ImageCache(object): t1 = i.copy() t1.thumbnail((size, size)) with BytesIO() as buf: - f1 = GzipFile(mode="wb", fileobj=buf) - t1.save(f1, format=i.format) - f1.close() + with GzipFile(mode="wb", fileobj=buf) as f1: + t1.save(f1, format=i.format) buf.seek(0) self.fs.put( buf, @@ -115,6 +114,30 @@ class ImageCache(object): kind=Kind.ACTOR_ICON.value, ) + def save_upload(self, obuf: BytesIO, filename: str) -> str: + # Remove EXIF metadata + if filename.lower().endswith(".jpg") or filename.lower().endswith(".jpeg"): + obuf.seek(0) + with BytesIO() as buf2: + piexif.remove(obuf.getvalue(), buf2) + obuf.truncate(0) + obuf.write(buf2.getvalue()) + + obuf.seek(0) + mtype = mimetypes.guess_type(filename)[0] + with BytesIO() as gbuf: + with GzipFile(mode="wb", fileobj=gbuf) as gzipfile: + gzipfile.write(obuf.getvalue()) + + gbuf.seek(0) + oid = self.fs.put( + gbuf, + content_type=mtype, + upload_filename=filename, + kind=Kind.UPLOAD.value, + ) + return str(oid) + def cache(self, url: str, kind: Kind) -> None: if kind == Kind.ACTOR_ICON: self.cache_actor_icon(url) From 10f26d035039d273fc8e66e0234a03c3f3e1aa10 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 6 Jul 2018 23:53:33 +0200 Subject: [PATCH 0178/1425] More pagination --- app.py | 117 ++++++++++++++++++++---------------------- templates/index.html | 16 ++---- templates/liked.html | 3 ++ templates/stream.html | 2 + templates/utils.html | 16 ++++++ 5 files changed, 81 insertions(+), 73 deletions(-) diff --git a/app.py b/app.py index 18dd98e..d45984e 100644 --- a/app.py +++ b/app.py @@ -588,33 +588,27 @@ def tmp_migrate3(): return "Done" -@app.route("/") -def index(): - if is_api_request(): - return jsonify(**ME) - +def paginated_query(db, q, limit=50, sort_key="_id"): older_than = newer_than = None query_sort = -1 - first_page = not request.args.get('older_than') and not request.args.get('newer_than') - limit = 5 - q = { - "box": Box.OUTBOX.value, - "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, - "activity.object.inReplyTo": None, - "meta.deleted": False, - "meta.undo": False, - } + first_page = not request.args.get("older_than") and not request.args.get( + "newer_than" + ) + query_older_than = request.args.get("older_than") query_newer_than = request.args.get("newer_than") + if query_older_than: q["_id"] = {"$lt": ObjectId(query_older_than)} elif query_newer_than: q["_id"] = {"$gt": ObjectId(query_newer_than)} query_sort = 1 - outbox_data = list(DB.activities.find(q, limit=limit+1).sort("_id", query_sort)) + outbox_data = list(db.find(q, limit=limit + 1).sort(sort_key, query_sort)) outbox_len = len(outbox_data) - outbox_data = sorted(outbox_data[:limit], key=lambda x: str(x["_id"]), reverse=True) + outbox_data = sorted( + outbox_data[:limit], key=lambda x: str(x[sort_key]), reverse=True + ) if query_older_than: newer_than = str(outbox_data[0]["_id"]) @@ -627,6 +621,23 @@ def index(): elif first_page and outbox_len == limit + 1: older_than = str(outbox_data[-1]["_id"]) + return outbox_data, older_than, newer_than + + +@app.route("/") +def index(): + if is_api_request(): + return jsonify(**ME) + + q = { + "box": Box.OUTBOX.value, + "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, + "activity.object.inReplyTo": None, + "meta.deleted": False, + "meta.undo": False, + } + outbox_data, older_than, newer_than = paginated_query(DB.activities, q) + return render_template( "index.html", outbox_data=outbox_data, @@ -637,24 +648,20 @@ def index(): @app.route("/with_replies") def with_replies(): - # FIXME(tsileo): implements pagination, also for the followers/following page - limit = 50 q = { "box": Box.OUTBOX.value, "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, "meta.deleted": False, "meta.undo": False, } - c = request.args.get("cursor") - if c: - q["_id"] = {"$lt": ObjectId(c)} + outbox_data, older_than, newer_than = paginated_query(DB.activities, q) - outbox_data = list(DB.activities.find(q, limit=limit).sort("_id", -1)) - cursor = None - if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]["_id"]) - - return render_template("index.html", outbox_data=outbox_data, cursor=cursor) + return render_template( + "index.html", + outbox_data=outbox_data, + older_than=older_than, + newer_than=newer_than, + ) def _build_thread(data, include_children=True): @@ -1107,8 +1114,6 @@ def new(): @app.route("/notifications") @login_required def notifications(): - # FIXME(tsileo): implements pagination, also for the followers/following page - limit = 50 # FIXME(tsileo): show unfollow (performed by the current actor) and liked??? mentions_query = { "type": ActivityType.CREATE.value, @@ -1141,16 +1146,14 @@ def notifications(): unfollow_query, ], } - c = request.args.get("cursor") - if c: - q["_id"] = {"$lt": ObjectId(c)} + inbox_data, older_than, newer_than = paginated_query(DB.activities, q) - outbox_data = list(DB.activities.find(q, limit=limit).sort("_id", -1)) - cursor = None - if outbox_data and len(outbox_data) == limit: - cursor = str(outbox_data[-1]["_id"]) - - return render_template("stream.html", inbox_data=outbox_data, cursor=cursor) + return render_template( + "stream.html", + inbox_data=inbox_data, + older_than=older_than, + newer_than=newer_than, + ) @app.route("/api/key") @@ -1254,26 +1257,19 @@ def api_undo(): @app.route("/stream") @login_required def stream(): - # FIXME(tsileo): implements pagination, also for the followers/following page - limit = 100 - c = request.args.get("cursor") q = { "box": Box.INBOX.value, "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, "meta.deleted": False, } - if c: - q["_id"] = {"$lt": ObjectId(c)} + inbox_data, older_than, newer_than = paginated_query(DB.activities, q) - inbox_data = list( - # FIXME(tsileo): reshape using meta.cached_object - DB.activities.find(q, limit=limit).sort("_id", -1) + return render_template( + "stream.html", + inbox_data=inbox_data, + older_than=older_than, + newer_than=newer_than, ) - cursor = None - if inbox_data and len(inbox_data) == limit: - cursor = str(inbox_data[-1]["_id"]) - - return render_template("stream.html", inbox_data=inbox_data, cursor=cursor) @app.route("/inbox", methods=["GET", "POST"]) @@ -1541,16 +1537,17 @@ def tags(tag): @app.route("/liked") def liked(): if not is_api_request(): + q = { + "box": Box.OUTBOX.value, + "type": ActivityType.LIKE.value, + "meta.deleted": False, + "meta.undo": False, + } + + liked, older_than, newer_than = paginated_query(DB.activities, q) + return render_template( - "liked.html", - liked=DB.activities.find( - { - "box": Box.OUTBOX.value, - "type": ActivityType.LIKE.value, - "meta.deleted": False, - "meta.undo": False, - } - ), + "liked.html", liked=liked, older_than=older_than, newer_than=newer_than ) q = {"meta.deleted": False, "meta.undo": False, "type": ActivityType.LIKE.value} diff --git a/templates/index.html b/templates/index.html index 5c2d178..115e65e 100644 --- a/templates/index.html +++ b/templates/index.html @@ -34,20 +34,10 @@ {% endif %} {% endfor %} -
    - {% if older_than %} - - {% endif %} - {% if newer_than %} - - {% endif %} + + {{ utils.display_pagination(older_than, newer_than) }}
    -
    {% endblock %} -{% block links %} -{% if older_than %}{% endif %} -{% if newer_than %}{% endif %} -{% endblock %} - +{% block links %}{{ utils.display_pagination_links(older_than, newer_than) }}{% endblock %} diff --git a/templates/liked.html b/templates/liked.html index b90462d..c8bf71e 100644 --- a/templates/liked.html +++ b/templates/liked.html @@ -12,7 +12,10 @@ {{ utils.display_note(item.meta.object) }} {% endif %} {% endfor %} + + {{ utils.display_pagination(older_than, newer_than) }}
    {% endblock %} +{% block links %}{{ utils.display_pagination_links(older_than, newer_than) }}{% endblock %} diff --git a/templates/stream.html b/templates/stream.html index f5c42a6..41ceecd 100644 --- a/templates/stream.html +++ b/templates/stream.html @@ -45,6 +45,8 @@ {% endif %} {% endfor %} + + {{ utils.display_pagination(older_than, newer_than) }}
    diff --git a/templates/utils.html b/templates/utils.html index 0b158c3..b56b9fc 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -168,3 +168,19 @@ {% endif %} {% endfor %} {% endmacro -%} + +{% macro display_pagination(older_than, newer_than) -%} +
    + {% if older_than %} + + {% endif %} + {% if newer_than %} + + {% endif %} +
    +{% endmacro -%} + +{% macro display_pagination_links(older_than, newer_than) -%} +{% if older_than %}{% endif %} +{% if newer_than %}{% endif %} +{% endmacro -%} From 6826e833fa6aad408eb675978bc3a19e53c5214b Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 6 Jul 2018 23:54:41 +0200 Subject: [PATCH 0179/1425] Change default page size to 25 --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index d45984e..69ca07d 100644 --- a/app.py +++ b/app.py @@ -588,7 +588,7 @@ def tmp_migrate3(): return "Done" -def paginated_query(db, q, limit=50, sort_key="_id"): +def paginated_query(db, q, limit=25, sort_key="_id"): older_than = newer_than = None query_sort = -1 first_page = not request.args.get("older_than") and not request.args.get( From 97fdcca06e8e798edb4d99c8ee4fcf811072296e Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 7 Jul 2018 00:08:44 +0200 Subject: [PATCH 0180/1425] Move admin stuff URL to /admin --- app.py | 25 +++++++++++++------------ templates/layout.html | 10 +++++----- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/app.py b/app.py index 69ca07d..5d92bac 100644 --- a/app.py +++ b/app.py @@ -390,6 +390,7 @@ def handle_activitypub_error(error): # App routes ROBOTS_TXT = """User-agent: * +Disallow: /login Disallow: /admin/ Disallow: /static/ Disallow: /media/ @@ -433,15 +434,15 @@ def serve_uploads(oid, fname): # Login -@app.route("/logout") +@app.route("/admin/logout") @login_required -def logout(): +def admin_logout(): session["logged_in"] = False return redirect("/") @app.route("/login", methods=["POST", "GET"]) -def login(): +def admin_login(): devices = [doc["device"] for doc in DB.u2f.find()] u2f_enabled = True if devices else False if request.method == "POST": @@ -461,7 +462,7 @@ def login(): session["challenge"] = None session["logged_in"] = True - return redirect(request.args.get("redirect") or "/admin") + return redirect(request.args.get("redirect") or "/notifications") else: abort(401) @@ -1063,9 +1064,9 @@ def outbox_activity_shares(item_id): ) -@app.route("/admin", methods=["GET"]) +@app.route("/admin/stats", methods=["GET"]) @login_required -def admin(): +def admin_stats(): q = { "meta.deleted": False, "meta.undo": False, @@ -1087,9 +1088,9 @@ def admin(): ) -@app.route("/new", methods=["GET"]) +@app.route("/admin/new", methods=["GET"]) @login_required -def new(): +def admin_new(): reply_id = None content = "" thread = [] @@ -1111,9 +1112,9 @@ def new(): return render_template("new.html", reply=reply_id, content=content, thread=thread) -@app.route("/notifications") +@app.route("/admin/notifications") @login_required -def notifications(): +def admin_notifications(): # FIXME(tsileo): show unfollow (performed by the current actor) and liked??? mentions_query = { "type": ActivityType.CREATE.value, @@ -1254,9 +1255,9 @@ def api_undo(): return _user_api_response(activity=undo.id) -@app.route("/stream") +@app.route("/admin/stream") @login_required -def stream(): +def admin_stream(): q = { "box": Box.INBOX.value, "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, diff --git a/templates/layout.html b/templates/layout.html index da804a3..dcf85ac 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -22,11 +22,11 @@ {% if logged_in %} {% endif %} From b31994dae7bee4c8bb80ca92b8be7e0a9560b12a Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 7 Jul 2018 00:12:35 +0200 Subject: [PATCH 0181/1425] Fix the admin redirect --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 5d92bac..dbff624 100644 --- a/app.py +++ b/app.py @@ -462,7 +462,7 @@ def admin_login(): session["challenge"] = None session["logged_in"] = True - return redirect(request.args.get("redirect") or "/notifications") + return redirect(request.args.get("redirect") or "/admin/notifications") else: abort(401) From 3281e2165eb1ab2c99f6c2a18b24eefda8ba8e86 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 7 Jul 2018 12:10:25 +0200 Subject: [PATCH 0182/1425] Bugfix for undo boosts/likes --- activitypub.py | 8 ++++++++ templates/index.html | 16 +++++++++++++++- templates/liked.html | 11 +++++++++++ templates/stream.html | 2 +- 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/activitypub.py b/activitypub.py index b891674..b5ed7c9 100644 --- a/activitypub.py +++ b/activitypub.py @@ -180,6 +180,7 @@ class MicroblogPubBackend(Backend): {"box": Box.OUTBOX.value, "activity.object.id": obj.id}, {"$inc": {"meta.count_like": -1}}, ) + DB.activities.update_one({"remote_id": like.id}, {"$set": {"meta.undo": True}}) @ensure_it_is_me def outbox_like(self, as_actor: ap.Person, like: ap.Like) -> None: @@ -199,6 +200,7 @@ class MicroblogPubBackend(Backend): {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": -1}, "$set": {"meta.liked": False}}, ) + DB.activities.update_one({"remote_id": like.id}, {"$set": {"meta.undo": True}}) @ensure_it_is_me def inbox_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: @@ -228,6 +230,9 @@ class MicroblogPubBackend(Backend): DB.activities.update_one( {"activity.object.id": obj.id}, {"$inc": {"meta.count_boost": -1}} ) + DB.activities.update_one( + {"remote_id": announce.id}, {"$set": {"meta.undo": True}} + ) @ensure_it_is_me def outbox_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: @@ -246,6 +251,9 @@ class MicroblogPubBackend(Backend): DB.activities.update_one( {"activity.object.id": obj.id}, {"$set": {"meta.boosted": False}} ) + DB.activities.update_one( + {"remote_id": announce.id}, {"$set": {"meta.undo": True}} + ) @ensure_it_is_me def inbox_delete(self, as_actor: ap.Person, delete: ap.Delete) -> None: diff --git a/templates/index.html b/templates/index.html index 115e65e..10ba52f 100644 --- a/templates/index.html +++ b/templates/index.html @@ -25,7 +25,21 @@ {% if item | has_type('Announce') %} {% set boost_actor = item.activity.actor | get_actor %} -

    {{ boost_actor.name }} boosted

    + {% if session.logged_in %} +
    + {{ boost_actor.name }} boosted + + + + + + +
    + {% else %} +

    + {{ boost_actor.name }} boosted +

    + {% endif %} {% if item.meta.object %} {{ utils.display_note(item.meta.object, ui=False) }} {% endif %} diff --git a/templates/liked.html b/templates/liked.html index c8bf71e..8255fdf 100644 --- a/templates/liked.html +++ b/templates/liked.html @@ -8,6 +8,17 @@
    {% for item in liked %} + {% if session.logged_in %} +
    +
    + + + + +
    +
    + + {% endif %} {% if item.meta.object %} {{ utils.display_note(item.meta.object) }} {% endif %} diff --git a/templates/stream.html b/templates/stream.html index 41ceecd..26fdc3b 100644 --- a/templates/stream.html +++ b/templates/stream.html @@ -9,7 +9,7 @@
    {% for item in inbox_data %} {% if item | has_type('Create') %} - {{ utils.display_note(item.activity.object, ui=True) }} + {{ utils.display_note(item.activity.object, ui=True, meta=item.meta) }} {% else %} {% if item | has_type('Announce') %} From 8ffeb1fe4f4c0da13ed49a5ff03957e4c1a5cf10 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 7 Jul 2018 12:53:32 +0200 Subject: [PATCH 0183/1425] Pre-cache the attachments --- activitypub.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/activitypub.py b/activitypub.py index b5ed7c9..55ba6ce 100644 --- a/activitypub.py +++ b/activitypub.py @@ -18,6 +18,8 @@ from config import ID from config import ME from config import USER_AGENT from config import USERNAME +from config import MEDIA_CACHE +from utils.media import Kind from little_boxes import activitypub as ap from little_boxes import strtobool from little_boxes.activitypub import _to_list @@ -87,6 +89,14 @@ class MicroblogPubBackend(Backend): } ) + # Generates thumbnails for the actor's icon and the attachments if any + actor = activity.get_actor() + if actor.icon: + MEDIA_CACHE.cache(actor.icon["url"], Kind.ACTOR_ICON) + if activity.type == ap.ActivityType.CREATE.value: + for attachment in activity.get_object()._data.get("attachment", []): + MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT) + @ensure_it_is_me def outbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: self.save(Box.OUTBOX, activity) From 285d2fa89083fc7fa675a520d59e94141f110687 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 7 Jul 2018 13:06:57 +0200 Subject: [PATCH 0184/1425] Filter the outbox when not authenticated --- app.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app.py b/app.py index dbff624..2a066de 100644 --- a/app.py +++ b/app.py @@ -886,12 +886,11 @@ def outbox(): if request.method == "GET": if not is_api_request(): abort(404) - # TODO(tsileo): filter the outbox if not authenticated - # FIXME(tsileo): filter deleted, add query support for build_ordered_collection + # TODO(tsileo): returns the whole outbox if authenticated q = { "box": Box.OUTBOX.value, - "meta.deleted": False, - # 'type': {'$in': [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, + "meta.deleted": False, # TODO(tsileo): retrieve deleted and expose tombstone + 'type': {'$in': [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, } return jsonify( **activitypub.build_ordered_collection( From 354a9ad13f1ba85624ccb8790eb8dd31805c3d16 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 7 Jul 2018 13:08:16 +0200 Subject: [PATCH 0185/1425] Also update the nodeinfo endpoint --- app.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 2a066de..05ce438 100644 --- a/app.py +++ b/app.py @@ -762,6 +762,11 @@ def note_by_id(note_id): @app.route("/nodeinfo") def nodeinfo(): + q = { + "box": Box.OUTBOX.value, + "meta.deleted": False, # TODO(tsileo): retrieve deleted and expose tombstone + 'type': {'$in': [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, + } return Response( headers={ "Content-Type": "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#" @@ -778,7 +783,7 @@ def nodeinfo(): "openRegistrations": False, "usage": { "users": {"total": 1}, - "localPosts": DB.activities.count({"box": Box.OUTBOX.value}), + "localPosts": DB.activities.count(q), }, "metadata": { "sourceCode": "https://github.com/tsileo/microblog.pub", From f7e6d37dce0f939b5463a955655e6dea6a9fff3e Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 7 Jul 2018 13:18:01 +0200 Subject: [PATCH 0186/1425] Re-add support for "extra inboxes" This allow to start posting public activities to other instances without follower. --- activitypub.py | 4 ++++ config.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/activitypub.py b/activitypub.py index 55ba6ce..338abd8 100644 --- a/activitypub.py +++ b/activitypub.py @@ -19,6 +19,7 @@ from config import ME from config import USER_AGENT from config import USERNAME from config import MEDIA_CACHE +from config import EXTRA_INBOXES from utils.media import Kind from little_boxes import activitypub as ap from little_boxes import strtobool @@ -65,6 +66,9 @@ class MicroblogPubBackend(Backend): """Setup a custom user agent.""" return USER_AGENT + def extra_inboxes(self) -> List[str]: + return EXTRA_INBOXES + def base_url(self) -> str: """Base URL config.""" return BASE_URL diff --git a/config.py b/config.py index c145932..9e7b844 100644 --- a/config.py +++ b/config.py @@ -70,7 +70,7 @@ with open(os.path.join(KEY_DIR, "me.yml")) as f: SUMMARY = conf["summary"] ICON_URL = conf["icon_url"] PASS = conf["pass"] - PUBLIC_INSTANCES = conf.get("public_instances", []) + EXTRA_INBOXES = conf.get("extra_inboxes", []) # Theme-related config theme_conf = conf.get("theme", {}) From f6c26abccbe184f38f13f1c7f3ab20caee693bda Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 7 Jul 2018 13:56:00 +0200 Subject: [PATCH 0187/1425] Tweak the admin, finish the pagination Fixes #14 --- README.md | 2 - activitypub.py | 6 +- app.py | 134 +++++++++++++++++++++++++++++++-------- templates/admin.html | 10 +-- templates/followers.html | 2 + templates/following.html | 2 + templates/layout.html | 3 +- 7 files changed, 115 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 56897d1..338016a 100644 --- a/README.md +++ b/README.md @@ -50,8 +50,6 @@ microblog.pub implements an [ActivityPub](http://activitypub.rocks/) server, it implements both the client to server API and the federated server to server API. -Compatible with [Mastodon](https://github.com/tootsuite/mastodon) (which is not following the spec closely), but will drop OStatus messages. - Activities are verified using HTTP Signatures or by fetching the content on the remote server directly. ## Running your instance diff --git a/activitypub.py b/activitypub.py index 338abd8..0166339 100644 --- a/activitypub.py +++ b/activitypub.py @@ -14,19 +14,19 @@ from html2text import html2text import tasks from config import BASE_URL from config import DB +from config import EXTRA_INBOXES from config import ID from config import ME +from config import MEDIA_CACHE from config import USER_AGENT from config import USERNAME -from config import MEDIA_CACHE -from config import EXTRA_INBOXES -from utils.media import Kind from little_boxes import activitypub as ap from little_boxes import strtobool from little_boxes.activitypub import _to_list from little_boxes.backend import Backend from little_boxes.collection import parse_collection as ap_parse_collection from little_boxes.errors import Error +from utils.media import Kind logger = logging.getLogger(__name__) diff --git a/app.py b/app.py index 05ce438..49076ac 100644 --- a/app.py +++ b/app.py @@ -132,12 +132,23 @@ def inject_config(): "type": ActivityType.LIKE.value, } ) + followers_q = { + "box": Box.INBOX.value, + "type": ActivityType.FOLLOW.value, + "meta.undo": False, + } + following_q = { + "box": Box.OUTBOX.value, + "type": ActivityType.FOLLOW.value, + "meta.undo": False, + } + return dict( microblogpub_version=VERSION, config=config, logged_in=session.get("logged_in", False), - followers_count=DB.followers.count(), - following_count=DB.following.count(), + followers_count=DB.activities.count(followers_q), + following_count=DB.activities.count(following_q), notes_count=notes_count, liked_count=liked_count, with_replies_count=with_replies_count, @@ -499,7 +510,14 @@ def authorize_follow(): actor = get_actor_url(request.form.get("profile")) if not actor: abort(500) - if DB.following.find({"remote_actor": actor}).count() > 0: + + q = { + "box": Box.OUTBOX.value, + "type": ActivityType.FOLLOW.value, + "meta.undo": False, + "activity.object": actor, + } + if DB.activities.count(q) > 0: return redirect("/following") follow = ap.Follow(actor=MY_PERSON.id, object=actor) @@ -589,6 +607,38 @@ def tmp_migrate3(): return "Done" +@app.route("/migration3") +@login_required +def tmp_migrate4(): + for activity in DB.activities.find( + {"box": Box.OUTBOX.value, "type": ActivityType.UNDO.value} + ): + try: + activity = ap.parse_activity(activity["activity"]) + if activity.get_object().type == ActivityType.FOLLOW.value: + DB.activities.update_one( + {"remote_id": activity.get_object().id}, + {"$set": {"meta.undo": True}}, + ) + print(activity.get_object().to_dict()) + except Exception: + app.logger.exception("failed") + for activity in DB.activities.find( + {"box": Box.INBOX.value, "type": ActivityType.UNDO.value} + ): + try: + activity = ap.parse_activity(activity["activity"]) + if activity.get_object().type == ActivityType.FOLLOW.value: + DB.activities.update_one( + {"remote_id": activity.get_object().id}, + {"$set": {"meta.undo": True}}, + ) + print(activity.get_object().to_dict()) + except Exception: + app.logger.exception("failed") + return "Done" + + def paginated_query(db, q, limit=25, sort_key="_id"): older_than = newer_than = None query_sort = -1 @@ -765,7 +815,7 @@ def nodeinfo(): q = { "box": Box.OUTBOX.value, "meta.deleted": False, # TODO(tsileo): retrieve deleted and expose tombstone - 'type': {'$in': [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, + "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, } return Response( headers={ @@ -781,10 +831,7 @@ def nodeinfo(): "protocols": ["activitypub"], "services": {"inbound": [], "outbound": []}, "openRegistrations": False, - "usage": { - "users": {"total": 1}, - "localPosts": DB.activities.count(q), - }, + "usage": {"users": {"total": 1}, "localPosts": DB.activities.count(q)}, "metadata": { "sourceCode": "https://github.com/tsileo/microblog.pub", "nodeName": f"@{USERNAME}@{DOMAIN}", @@ -895,7 +942,7 @@ def outbox(): q = { "box": Box.OUTBOX.value, "meta.deleted": False, # TODO(tsileo): retrieve deleted and expose tombstone - 'type': {'$in': [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, + "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, } return jsonify( **activitypub.build_ordered_collection( @@ -1068,9 +1115,9 @@ def outbox_activity_shares(item_id): ) -@app.route("/admin/stats", methods=["GET"]) +@app.route("/admin", methods=["GET"]) @login_required -def admin_stats(): +def admin(): q = { "meta.deleted": False, "meta.undo": False, @@ -1084,11 +1131,21 @@ def admin_stats(): instances=list(DB.instances.find()), inbox_size=DB.activities.count({"box": Box.INBOX.value}), outbox_size=DB.activities.count({"box": Box.OUTBOX.value}), - object_cache_size=0, - actor_cache_size=0, col_liked=col_liked, - col_followers=DB.followers.count(), - col_following=DB.following.count(), + col_followers=DB.activities.count( + { + "box": Box.INBOX.value, + "type": ActivityType.FOLLOW.value, + "meta.undo": False, + } + ), + col_following=DB.activities.count( + { + "box": Box.OUTBOX.value, + "type": ActivityType.FOLLOW.value, + "meta.undo": False, + } + ), ) @@ -1452,7 +1509,14 @@ def api_block(): def api_follow(): actor = _user_api_arg("actor") - existing = DB.following.find_one({"remote_actor": actor}) + q = { + "box": Box.OUTBOX.value, + "type": ActivityType.FOLLOW.value, + "meta.undo": False, + "activity.object": actor, + } + + existing = DB.activities.find_one(q) if existing: return _user_api_response(activity=existing["activity"]["id"]) @@ -1464,36 +1528,50 @@ def api_follow(): @app.route("/followers") def followers(): + q = {"box": Box.INBOX.value, "type": ActivityType.FOLLOW.value, "meta.undo": False} + if is_api_request(): return jsonify( **activitypub.build_ordered_collection( - DB.followers, + DB.activities, + q=q, cursor=request.args.get("cursor"), - map_func=lambda doc: doc["remote_actor"], + map_func=lambda doc: doc["activity"]["object"], ) ) - followers = [ - ACTOR_SERVICE.get(doc["remote_actor"]) for doc in DB.followers.find(limit=50) - ] - return render_template("followers.html", followers_data=followers) + followers, older_than, newer_than = paginated_query(DB.activities, q) + followers = [ACTOR_SERVICE.get(doc["activity"]["object"]) for doc in followers] + return render_template( + "followers.html", + followers_data=followers, + older_than=older_than, + newer_than=newer_than, + ) @app.route("/following") def following(): + q = {"box": Box.OUTBOX.value, "type": ActivityType.FOLLOW.value, "meta.undo": False} + if is_api_request(): return jsonify( **activitypub.build_ordered_collection( - DB.following, + DB.activities, + q=q, cursor=request.args.get("cursor"), - map_func=lambda doc: doc["remote_actor"], + map_func=lambda doc: doc["activity"]["object"], ) ) - following = [ - ACTOR_SERVICE.get(doc["remote_actor"]) for doc in DB.following.find(limit=50) - ] - return render_template("following.html", following_data=following) + following, older_than, newer_than = paginated_query(DB.activities, q) + following = [ACTOR_SERVICE.get(doc["activity"]["object"]) for doc in following] + return render_template( + "following.html", + following_data=following, + older_than=older_than, + newer_than=newer_than, + ) @app.route("/tags/") diff --git a/templates/admin.html b/templates/admin.html index e573e02..d2e1e95 100644 --- a/templates/admin.html +++ b/templates/admin.html @@ -5,13 +5,11 @@
    {% include "header.html" %}
    -

    Stats

    +

    Admin

    DB

    • Inbox size: {{ inbox_size }}
    • Outbox size: {{ outbox_size }}
      • -
    • Object cache size: {{ object_cache_size }}
      • -
    • Actor cache size: {{ actor_cache_size }}

    Collections

      @@ -19,12 +17,6 @@
    • following: {{ col_following }}
    • liked: {{col_liked }}
    -

    Known Instances

    -
    diff --git a/templates/followers.html b/templates/followers.html index e08ad2a..fe15e62 100644 --- a/templates/followers.html +++ b/templates/followers.html @@ -13,7 +13,9 @@ {{ utils.display_actor_inline(follower, size=80) }}
    {% endfor %} + {{ utils.display_pagination(older_than, newer_than) }}
    {% endblock %} +{% block links %}{{ utils.display_pagination_links(older_than, newer_than) }}{% endblock %} diff --git a/templates/following.html b/templates/following.html index 0e6efb4..3dc283c 100644 --- a/templates/following.html +++ b/templates/following.html @@ -13,7 +13,9 @@ {{ utils.display_actor_inline(followed, size=80) }}
    {% endfor %} + {{ utils.display_pagination(older_than, newer_than) }}
    {% endblock %} +{% block links %}{{ utils.display_pagination_links(older_than, newer_than) }}{% endblock %} diff --git a/templates/layout.html b/templates/layout.html index dcf85ac..d36d11d 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -21,11 +21,10 @@ {% if logged_in %} {% endif %} From 9ff6aee0ae83582421fc74eee06028a7bb2ef1ed Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 7 Jul 2018 14:04:12 +0200 Subject: [PATCH 0188/1425] Tweak the CI delay --- tests/federation_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/federation_test.py b/tests/federation_test.py index 6e0a7ea..49f331f 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -20,7 +20,7 @@ class Instance(object): def __init__(self, name, host_url, docker_url=None): self.host_url = host_url self.docker_url = docker_url or host_url - self._create_delay = 10 + self._create_delay = 15 with open( os.path.join( os.path.dirname(os.path.abspath(__file__)), From bf208d04226eb5d61f1add34d007b411385cd52e Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 7 Jul 2018 14:07:29 +0200 Subject: [PATCH 0189/1425] Fix followers --- app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 49076ac..e5b4938 100644 --- a/app.py +++ b/app.py @@ -1536,12 +1536,12 @@ def followers(): DB.activities, q=q, cursor=request.args.get("cursor"), - map_func=lambda doc: doc["activity"]["object"], + map_func=lambda doc: doc["activity"]["actor"], ) ) followers, older_than, newer_than = paginated_query(DB.activities, q) - followers = [ACTOR_SERVICE.get(doc["activity"]["object"]) for doc in followers] + followers = [ACTOR_SERVICE.get(doc["activity"]["actor"]) for doc in followers] return render_template( "followers.html", followers_data=followers, From 20a40ee34482357a10aa26bed67c48640bf860b1 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 7 Jul 2018 14:07:53 +0200 Subject: [PATCH 0190/1425] Tweak the CI delay --- tests/federation_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/federation_test.py b/tests/federation_test.py index 49f331f..f926812 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -20,7 +20,7 @@ class Instance(object): def __init__(self, name, host_url, docker_url=None): self.host_url = host_url self.docker_url = docker_url or host_url - self._create_delay = 15 + self._create_delay = 12 with open( os.path.join( os.path.dirname(os.path.abspath(__file__)), From 5ad4d8173530dad4e147f0216f066e874cfca520 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 7 Jul 2018 14:11:23 +0200 Subject: [PATCH 0191/1425] Fix unfollow --- activitypub.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/activitypub.py b/activitypub.py index 0166339..a62b35c 100644 --- a/activitypub.py +++ b/activitypub.py @@ -156,26 +156,19 @@ class MicroblogPubBackend(Backend): @ensure_it_is_me def new_follower(self, as_actor: ap.Person, follow: ap.Follow) -> None: - remote_actor = follow.get_actor().id - - if DB.followers.find({"remote_actor": remote_actor}).count() == 0: - DB.followers.insert_one({"remote_actor": remote_actor}) + pass @ensure_it_is_me def undo_new_follower(self, as_actor: ap.Person, follow: ap.Follow) -> None: - # TODO(tsileo): update the follow to set undo - DB.followers.delete_one({"remote_actor": follow.get_actor().id}) + DB.activities.update_one({"remote_id": follow.id}, {"$set": {"meta.undo": True}}) @ensure_it_is_me def undo_new_following(self, as_actor: ap.Person, follow: ap.Follow) -> None: - # TODO(tsileo): update the follow to set undo - DB.following.delete_one({"remote_actor": follow.get_object().id}) + DB.activities.update_one({"remote_id": follow.id}, {"$set": {"meta.undo": True}}) @ensure_it_is_me def new_following(self, as_actor: ap.Person, follow: ap.Follow) -> None: - remote_actor = follow.get_object().id - if DB.following.find({"remote_actor": remote_actor}).count() == 0: - DB.following.insert_one({"remote_actor": remote_actor}) + pass @ensure_it_is_me def inbox_like(self, as_actor: ap.Person, like: ap.Like) -> None: From da029c1566c0aaf1b3e49aa1dd12d96db18773bf Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 8 Jul 2018 12:24:49 +0200 Subject: [PATCH 0192/1425] Fix collection parsing --- activitypub.py | 45 ++++++++++++++++++++++++++++----------------- app.py | 5 ++--- config.py | 4 ++++ 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/activitypub.py b/activitypub.py index a62b35c..355d1f9 100644 --- a/activitypub.py +++ b/activitypub.py @@ -24,7 +24,6 @@ from little_boxes import activitypub as ap from little_boxes import strtobool from little_boxes.activitypub import _to_list from little_boxes.backend import Backend -from little_boxes.collection import parse_collection as ap_parse_collection from little_boxes.errors import Error from utils.media import Kind @@ -105,6 +104,28 @@ class MicroblogPubBackend(Backend): def outbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: self.save(Box.OUTBOX, activity) + def parse_collection( + payload: Optional[Dict[str, Any]] = None, url: Optional[str] = None + ) -> List[str]: + """Resolve/fetch a `Collection`/`OrderedCollection`.""" + # Resolve internal collections via MongoDB directly + if url == ID + "/followers": + q = { + "box": Box.INBOX.value, + "type": ap.ActivityType.FOLLOW.value, + "meta.undo": False, + } + return [doc["activity"]["actor"] for doc in DB.activities.find(q)] + elif url == ID + "/following": + q = { + "box": Box.OUTBOX.value, + "type": ap.ActivityType.FOLLOW.value, + "meta.undo": False, + } + return [doc["activity"]["object"] for doc in DB.activities.find(q)] + + return super().parse_collection(payload, url) + @ensure_it_is_me def outbox_is_blocked(self, as_actor: ap.Person, actor_id: str) -> bool: return bool( @@ -160,11 +181,15 @@ class MicroblogPubBackend(Backend): @ensure_it_is_me def undo_new_follower(self, as_actor: ap.Person, follow: ap.Follow) -> None: - DB.activities.update_one({"remote_id": follow.id}, {"$set": {"meta.undo": True}}) + DB.activities.update_one( + {"remote_id": follow.id}, {"$set": {"meta.undo": True}} + ) @ensure_it_is_me def undo_new_following(self, as_actor: ap.Person, follow: ap.Follow) -> None: - DB.activities.update_one({"remote_id": follow.id}, {"$set": {"meta.undo": True}}) + DB.activities.update_one( + {"remote_id": follow.id}, {"$set": {"meta.undo": True}} + ) @ensure_it_is_me def new_following(self, as_actor: ap.Person, follow: ap.Follow) -> None: @@ -498,20 +523,6 @@ def build_inbox_json_feed( return resp -def parse_collection( - payload: Optional[Dict[str, Any]] = None, url: Optional[str] = None -) -> List[str]: - """Resolve/fetch a `Collection`/`OrderedCollection`.""" - # Resolve internal collections via MongoDB directly - if url == ID + "/followers": - return [doc["remote_actor"] for doc in DB.followers.find()] - elif url == ID + "/following": - return [doc["remote_actor"] for doc in DB.following.find()] - - # Go through all the pages - return ap_parse_collection(payload, url) - - def embed_collection(total_items, first_page_id): """Helper creating a root OrderedCollection with a link to the first page.""" return { diff --git a/app.py b/app.py index e5b4938..b4748c8 100644 --- a/app.py +++ b/app.py @@ -41,6 +41,7 @@ import activitypub import config from activitypub import Box from activitypub import embed_collection +from config import ACTOR_SERVICE from config import ADMIN_API_KEY from config import BASE_URL from config import DB @@ -53,6 +54,7 @@ from config import JWT from config import KEY from config import ME from config import MEDIA_CACHE +from config import OBJECT_SERVICE from config import PASS from config import USERNAME from config import VERSION @@ -73,9 +75,6 @@ from little_boxes.webfinger import get_actor_url from little_boxes.webfinger import get_remote_follow_template from utils.key import get_secret_key from utils.media import Kind -from utils.object_service import ObjectService - -OBJECT_SERVICE = ACTOR_SERVICE = ObjectService() back = activitypub.MicroblogPubBackend() ap.use_backend(back) diff --git a/config.py b/config.py index 9e7b844..abb2187 100644 --- a/config.py +++ b/config.py @@ -15,6 +15,7 @@ from utils.key import KEY_DIR from utils.key import get_key from utils.key import get_secret_key from utils.media import MediaCache +from utils.object_service import ObjectService class ThemeStyle(Enum): @@ -147,3 +148,6 @@ ME = { }, "publicKey": KEY.to_dict(), } + + +OBJECT_SERVICE = ACTOR_SERVICE = ObjectService() From c573f48873ca37a7a9871d65be6877f49b1ef0be Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 8 Jul 2018 12:31:59 +0200 Subject: [PATCH 0193/1425] Fix replying in the admin UI Fixes #22 --- app.py | 2 +- templates/utils.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index b4748c8..322e1e9 100644 --- a/app.py +++ b/app.py @@ -472,7 +472,7 @@ def admin_login(): session["challenge"] = None session["logged_in"] = True - return redirect(request.args.get("redirect") or "/admin/notifications") + return redirect(request.args.get("redirect") or url_for("admin_notifications")) else: abort(401) diff --git a/templates/utils.html b/templates/utils.html index b56b9fc..2ba93ce 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -72,7 +72,7 @@ {% if ui and session.logged_in %} {% set aid = obj.id | quote_plus %} -reply +reply {% set perma_id = obj.id | permalink_id %} {% set redir = request.path + "#activity-" + perma_id %} From 3d3e9da8007702d2e4a09d1c8f10d5ddf63a8d13 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 8 Jul 2018 12:43:34 +0200 Subject: [PATCH 0194/1425] Bugfix collection parsing --- activitypub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitypub.py b/activitypub.py index 355d1f9..b30a98c 100644 --- a/activitypub.py +++ b/activitypub.py @@ -105,7 +105,7 @@ class MicroblogPubBackend(Backend): self.save(Box.OUTBOX, activity) def parse_collection( - payload: Optional[Dict[str, Any]] = None, url: Optional[str] = None + self, payload: Optional[Dict[str, Any]] = None, url: Optional[str] = None ) -> List[str]: """Resolve/fetch a `Collection`/`OrderedCollection`.""" # Resolve internal collections via MongoDB directly From 7b6982c959098d9df1a0e6aafb78e28140232d72 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 9 Jul 2018 22:29:03 +0200 Subject: [PATCH 0195/1425] Better Delete handling --- activitypub.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/activitypub.py b/activitypub.py index b30a98c..37b7ef1 100644 --- a/activitypub.py +++ b/activitypub.py @@ -305,6 +305,12 @@ class MicroblogPubBackend(Backend): ).get_object() logger.info(f"inbox_delete handle_replies obj={obj!r}") + + # Fake a Undo so any related Like/Announce doesn't appear on the web UI + DB.activities.update( + {"meta.object.id": obj.id}, + {"$set": {"meta.undo": True, "meta.exta": "object deleted"}}, + ) if obj: self._handle_replies_delete(as_actor, obj) @@ -325,6 +331,11 @@ class MicroblogPubBackend(Backend): )["activity"] ).get_object() + DB.activities.update( + {"meta.object.id": obj.id}, + {"$set": {"meta.undo": True, "meta.exta": "object deleted"}}, + ) + self._handle_replies_delete(as_actor, obj) @ensure_it_is_me From 8ae3f1e3a3e34fa1dd54c55a77350bfef4dc615e Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 10 Jul 2018 00:25:26 +0200 Subject: [PATCH 0196/1425] Tweak the tombstone support --- activitypub.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/activitypub.py b/activitypub.py index 37b7ef1..f591a5b 100644 --- a/activitypub.py +++ b/activitypub.py @@ -25,6 +25,7 @@ from little_boxes import strtobool from little_boxes.activitypub import _to_list from little_boxes.backend import Backend from little_boxes.errors import Error +from little_boxes.errors import ActivityGoneError from utils.media import Kind logger = logging.getLogger(__name__) @@ -150,6 +151,8 @@ class MicroblogPubBackend(Backend): iri = iri.replace("/activity", "") is_a_note = True data = DB.activities.find_one({"box": Box.OUTBOX.value, "remote_id": iri}) + if data and data["meta"]["deleted"]: + raise ActivityGoneError(f"{iri} is gone") if data and is_a_note: return data["activity"]["object"] elif data: @@ -158,6 +161,8 @@ class MicroblogPubBackend(Backend): # Check if the activity is stored in the inbox data = DB.activities.find_one({"remote_id": iri}) if data: + if data["meta"]["deleted"]: + raise ActivityGoneError(f"{iri} is gone") return data["activity"] # Fetch the URL via HTTP From 108b957dcc8212faff545d42e421e4aff09eb024 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 10 Jul 2018 00:49:52 +0200 Subject: [PATCH 0197/1425] More work on Tombstone handling --- activitypub.py | 2 +- app.py | 27 +++++++++++++++------------ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/activitypub.py b/activitypub.py index f591a5b..336ede5 100644 --- a/activitypub.py +++ b/activitypub.py @@ -24,8 +24,8 @@ from little_boxes import activitypub as ap from little_boxes import strtobool from little_boxes.activitypub import _to_list from little_boxes.backend import Backend -from little_boxes.errors import Error from little_boxes.errors import ActivityGoneError +from little_boxes.errors import Error from utils.media import Kind logger = logging.getLogger(__name__) diff --git a/app.py b/app.py index 322e1e9..a734e8e 100644 --- a/app.py +++ b/app.py @@ -472,7 +472,9 @@ def admin_login(): session["challenge"] = None session["logged_in"] = True - return redirect(request.args.get("redirect") or url_for("admin_notifications")) + return redirect( + request.args.get("redirect") or url_for("admin_notifications") + ) else: abort(401) @@ -940,7 +942,7 @@ def outbox(): # TODO(tsileo): returns the whole outbox if authenticated q = { "box": Box.OUTBOX.value, - "meta.deleted": False, # TODO(tsileo): retrieve deleted and expose tombstone + "meta.deleted": False, "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, } return jsonify( @@ -974,9 +976,12 @@ def outbox_detail(item_id): doc = DB.activities.find_one( {"box": Box.OUTBOX.value, "remote_id": back.activity_url(item_id)} ) + if not doc: + abort(404) + if doc["meta"].get("deleted", False): obj = ap.parse_activity(doc["activity"]) - resp = jsonify(**obj.get_object().get_tombstone()) + resp = jsonify(**obj.get_tombstone().to_dict()) resp.status_code = 410 return resp return jsonify(**activity_from_doc(doc)) @@ -984,17 +989,18 @@ def outbox_detail(item_id): @app.route("/outbox//activity") def outbox_activity(item_id): - # TODO(tsileo): handle Tombstone data = DB.activities.find_one( - { - "box": Box.OUTBOX.value, - "remote_id": back.activity_url(item_id), - "meta.deleted": False, - } + {"box": Box.OUTBOX.value, "remote_id": back.activity_url(item_id)} ) if not data: abort(404) obj = activity_from_doc(data) + if data["meta"].get("deleted", False): + obj = ap.parse_activity(data["activity"]) + resp = jsonify(**obj.get_object().get_tombstone().to_dict()) + resp.status_code = 410 + return resp + if obj["type"] != ActivityType.CREATE.value: abort(404) return jsonify(**obj["object"]) @@ -1002,7 +1008,6 @@ def outbox_activity(item_id): @app.route("/outbox//replies") def outbox_activity_replies(item_id): - # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) data = DB.activities.find_one( @@ -1038,7 +1043,6 @@ def outbox_activity_replies(item_id): @app.route("/outbox//likes") def outbox_activity_likes(item_id): - # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) data = DB.activities.find_one( @@ -1077,7 +1081,6 @@ def outbox_activity_likes(item_id): @app.route("/outbox//shares") def outbox_activity_shares(item_id): - # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) data = DB.activities.find_one( From 5a13e4b362da42951ba512399f720c10c385c07f Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 10 Jul 2018 01:06:32 +0200 Subject: [PATCH 0198/1425] Tombstome handling (#23) * Better Delete handling * Tweak the tombstone support * More work on Tombstone handling --- activitypub.py | 16 ++++++++++++++++ app.py | 27 +++++++++++++++------------ 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/activitypub.py b/activitypub.py index b30a98c..336ede5 100644 --- a/activitypub.py +++ b/activitypub.py @@ -24,6 +24,7 @@ from little_boxes import activitypub as ap from little_boxes import strtobool from little_boxes.activitypub import _to_list from little_boxes.backend import Backend +from little_boxes.errors import ActivityGoneError from little_boxes.errors import Error from utils.media import Kind @@ -150,6 +151,8 @@ class MicroblogPubBackend(Backend): iri = iri.replace("/activity", "") is_a_note = True data = DB.activities.find_one({"box": Box.OUTBOX.value, "remote_id": iri}) + if data and data["meta"]["deleted"]: + raise ActivityGoneError(f"{iri} is gone") if data and is_a_note: return data["activity"]["object"] elif data: @@ -158,6 +161,8 @@ class MicroblogPubBackend(Backend): # Check if the activity is stored in the inbox data = DB.activities.find_one({"remote_id": iri}) if data: + if data["meta"]["deleted"]: + raise ActivityGoneError(f"{iri} is gone") return data["activity"] # Fetch the URL via HTTP @@ -305,6 +310,12 @@ class MicroblogPubBackend(Backend): ).get_object() logger.info(f"inbox_delete handle_replies obj={obj!r}") + + # Fake a Undo so any related Like/Announce doesn't appear on the web UI + DB.activities.update( + {"meta.object.id": obj.id}, + {"$set": {"meta.undo": True, "meta.exta": "object deleted"}}, + ) if obj: self._handle_replies_delete(as_actor, obj) @@ -325,6 +336,11 @@ class MicroblogPubBackend(Backend): )["activity"] ).get_object() + DB.activities.update( + {"meta.object.id": obj.id}, + {"$set": {"meta.undo": True, "meta.exta": "object deleted"}}, + ) + self._handle_replies_delete(as_actor, obj) @ensure_it_is_me diff --git a/app.py b/app.py index 322e1e9..a734e8e 100644 --- a/app.py +++ b/app.py @@ -472,7 +472,9 @@ def admin_login(): session["challenge"] = None session["logged_in"] = True - return redirect(request.args.get("redirect") or url_for("admin_notifications")) + return redirect( + request.args.get("redirect") or url_for("admin_notifications") + ) else: abort(401) @@ -940,7 +942,7 @@ def outbox(): # TODO(tsileo): returns the whole outbox if authenticated q = { "box": Box.OUTBOX.value, - "meta.deleted": False, # TODO(tsileo): retrieve deleted and expose tombstone + "meta.deleted": False, "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, } return jsonify( @@ -974,9 +976,12 @@ def outbox_detail(item_id): doc = DB.activities.find_one( {"box": Box.OUTBOX.value, "remote_id": back.activity_url(item_id)} ) + if not doc: + abort(404) + if doc["meta"].get("deleted", False): obj = ap.parse_activity(doc["activity"]) - resp = jsonify(**obj.get_object().get_tombstone()) + resp = jsonify(**obj.get_tombstone().to_dict()) resp.status_code = 410 return resp return jsonify(**activity_from_doc(doc)) @@ -984,17 +989,18 @@ def outbox_detail(item_id): @app.route("/outbox//activity") def outbox_activity(item_id): - # TODO(tsileo): handle Tombstone data = DB.activities.find_one( - { - "box": Box.OUTBOX.value, - "remote_id": back.activity_url(item_id), - "meta.deleted": False, - } + {"box": Box.OUTBOX.value, "remote_id": back.activity_url(item_id)} ) if not data: abort(404) obj = activity_from_doc(data) + if data["meta"].get("deleted", False): + obj = ap.parse_activity(data["activity"]) + resp = jsonify(**obj.get_object().get_tombstone().to_dict()) + resp.status_code = 410 + return resp + if obj["type"] != ActivityType.CREATE.value: abort(404) return jsonify(**obj["object"]) @@ -1002,7 +1008,6 @@ def outbox_activity(item_id): @app.route("/outbox//replies") def outbox_activity_replies(item_id): - # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) data = DB.activities.find_one( @@ -1038,7 +1043,6 @@ def outbox_activity_replies(item_id): @app.route("/outbox//likes") def outbox_activity_likes(item_id): - # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) data = DB.activities.find_one( @@ -1077,7 +1081,6 @@ def outbox_activity_likes(item_id): @app.route("/outbox//shares") def outbox_activity_shares(item_id): - # TODO(tsileo): handle Tombstone if not is_api_request(): abort(404) data = DB.activities.find_one( From 6aee810a1f207023ae5b38c47beaa4e068d731ee Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 10 Jul 2018 23:02:06 +0200 Subject: [PATCH 0199/1425] Handle deleted actors in the notifications --- app.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index a734e8e..6c1dfa8 100644 --- a/app.py +++ b/app.py @@ -67,6 +67,7 @@ from little_boxes.activitypub import clean_activity from little_boxes.activitypub import get_backend from little_boxes.content_helper import parse_markdown from little_boxes.errors import ActivityNotFoundError +from little_boxes.errors import ActivityGoneError from little_boxes.errors import Error from little_boxes.errors import NotFromOutboxError from little_boxes.httpsig import HTTPSigAuth @@ -265,8 +266,10 @@ def get_actor(url): if not url: return None print(f"GET_ACTOR {url}") - return ACTOR_SERVICE.get(url) - + try: + return ACTOR_SERVICE.get(url) + except (ActivityNotFoundError, ActivityGoneError): + return f"Deleted<{url}>" @app.template_filter() def format_time(val): From c9a10bddb4c69a27c36fa37540d7a817902f54ba Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 10 Jul 2018 23:04:05 +0200 Subject: [PATCH 0200/1425] Formatting --- app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 6c1dfa8..e58be68 100644 --- a/app.py +++ b/app.py @@ -66,8 +66,8 @@ from little_boxes.activitypub import _to_list from little_boxes.activitypub import clean_activity from little_boxes.activitypub import get_backend from little_boxes.content_helper import parse_markdown -from little_boxes.errors import ActivityNotFoundError from little_boxes.errors import ActivityGoneError +from little_boxes.errors import ActivityNotFoundError from little_boxes.errors import Error from little_boxes.errors import NotFromOutboxError from little_boxes.httpsig import HTTPSigAuth @@ -271,6 +271,7 @@ def get_actor(url): except (ActivityNotFoundError, ActivityGoneError): return f"Deleted<{url}>" + @app.template_filter() def format_time(val): if val: From 524a63235dca1efac0f11bad2be95972bc1148e8 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 10 Jul 2018 23:19:00 +0200 Subject: [PATCH 0201/1425] Add the alternate link for all activitypub stuff in the templates --- templates/followers.html | 4 +++- templates/following.html | 4 +++- templates/index.html | 6 +++++- templates/layout.html | 5 ----- templates/liked.html | 4 +++- templates/note.html | 1 + templates/tags.html | 3 +++ 7 files changed, 18 insertions(+), 9 deletions(-) diff --git a/templates/followers.html b/templates/followers.html index fe15e62..b492e29 100644 --- a/templates/followers.html +++ b/templates/followers.html @@ -18,4 +18,6 @@
    {% endblock %} -{% block links %}{{ utils.display_pagination_links(older_than, newer_than) }}{% endblock %} +{% block links %} + +{{ utils.display_pagination_links(older_than, newer_than) }}{% endblock %} diff --git a/templates/following.html b/templates/following.html index 3dc283c..c249b7d 100644 --- a/templates/following.html +++ b/templates/following.html @@ -18,4 +18,6 @@
    {% endblock %} -{% block links %}{{ utils.display_pagination_links(older_than, newer_than) }}{% endblock %} +{% block links %} + +{{ utils.display_pagination_links(older_than, newer_than) }}{% endblock %} diff --git a/templates/index.html b/templates/index.html index 10ba52f..6cc2e37 100644 --- a/templates/index.html +++ b/templates/index.html @@ -54,4 +54,8 @@
    {% endblock %} -{% block links %}{{ utils.display_pagination_links(older_than, newer_than) }}{% endblock %} +{% block links %} + + + +{{ utils.display_pagination_links(older_than, newer_than) }}{% endblock %} diff --git a/templates/layout.html b/templates/layout.html index d36d11d..21522fa 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -8,12 +8,7 @@ - {% if not request.args.get("older_than") and not request.args.get("previous_than") %}{% endif %} - - - - {% block links %}{% endblock %} {% if config.THEME_COLOR %}{% endif %} diff --git a/templates/liked.html b/templates/liked.html index 8255fdf..29b903f 100644 --- a/templates/liked.html +++ b/templates/liked.html @@ -29,4 +29,6 @@
    {% endblock %} -{% block links %}{{ utils.display_pagination_links(older_than, newer_than) }}{% endblock %} +{% block links %} + +{{ utils.display_pagination_links(older_than, newer_than) }}{% endblock %} diff --git a/templates/note.html b/templates/note.html index 8599778..053b6c7 100644 --- a/templates/note.html +++ b/templates/note.html @@ -19,3 +19,4 @@ {{ utils.display_thread(thread, likes=likes, shares=shares) }}
    {% endblock %} +{% block links %}{% endblock %} diff --git a/templates/tags.html b/templates/tags.html index 3beaa8b..e0ed839 100644 --- a/templates/tags.html +++ b/templates/tags.html @@ -29,3 +29,6 @@
    {% endblock %} +{% block links %} + +{% endblock %} From 01e165640e28fe5b202b34d8dc50392e32fdd9f9 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 11 Jul 2018 19:38:24 +0200 Subject: [PATCH 0202/1425] Remote the object service, use the backend instead --- app.py | 16 +++++++--------- config.py | 4 ---- utils/object_service.py | 21 --------------------- 3 files changed, 7 insertions(+), 34 deletions(-) delete mode 100644 utils/object_service.py diff --git a/app.py b/app.py index e58be68..e4f4995 100644 --- a/app.py +++ b/app.py @@ -41,7 +41,6 @@ import activitypub import config from activitypub import Box from activitypub import embed_collection -from config import ACTOR_SERVICE from config import ADMIN_API_KEY from config import BASE_URL from config import DB @@ -54,7 +53,6 @@ from config import JWT from config import KEY from config import ME from config import MEDIA_CACHE -from config import OBJECT_SERVICE from config import PASS from config import USERNAME from config import VERSION @@ -267,7 +265,7 @@ def get_actor(url): return None print(f"GET_ACTOR {url}") try: - return ACTOR_SERVICE.get(url) + return get_backend().fetch_iri(url) except (ActivityNotFoundError, ActivityGoneError): return f"Deleted<{url}>" @@ -794,7 +792,7 @@ def note_by_id(note_id): } ) ) - likes = [ACTOR_SERVICE.get(doc["activity"]["actor"]) for doc in likes] + likes = [get_backend().fetch_iri(doc["activity"]["actor"]) for doc in likes] shares = list( DB.activities.find( @@ -808,7 +806,7 @@ def note_by_id(note_id): } ) ) - shares = [ACTOR_SERVICE.get(doc["activity"]["actor"]) for doc in shares] + shares = [get_backend().fetch_iri(doc["activity"]["actor"]) for doc in shares] return render_template( "note.html", likes=likes, shares=shares, thread=thread, note=data @@ -1248,7 +1246,7 @@ def _user_api_arg(key: str, **kwargs): def _user_api_get_note(from_outbox: bool = False): oid = _user_api_arg("id") - note = ap.parse_activity(OBJECT_SERVICE.get(oid), expected=ActivityType.NOTE) + note = ap.parse_activity(get_backend().fetch_iri(oid), expected=ActivityType.NOTE) if from_outbox and not note.id.startswith(ID): raise NotFromOutboxError( f"cannot load {note.id}, id must be owned by the server" @@ -1436,7 +1434,7 @@ def api_new_note(): cc = [ID + "/followers"] if _reply: - reply = ap.parse_activity(OBJECT_SERVICE.get(_reply)) + reply = ap.fetch_remote_activity(_reply) cc.append(reply.attributedTo) for tag in tags: @@ -1547,7 +1545,7 @@ def followers(): ) followers, older_than, newer_than = paginated_query(DB.activities, q) - followers = [ACTOR_SERVICE.get(doc["activity"]["actor"]) for doc in followers] + followers = [get_backend().fetch_iri(doc["activity"]["actor"]) for doc in followers] return render_template( "followers.html", followers_data=followers, @@ -1571,7 +1569,7 @@ def following(): ) following, older_than, newer_than = paginated_query(DB.activities, q) - following = [ACTOR_SERVICE.get(doc["activity"]["object"]) for doc in following] + following = [get_backend.fetch_iri(doc["activity"]["object"]) for doc in following] return render_template( "following.html", following_data=following, diff --git a/config.py b/config.py index abb2187..9e7b844 100644 --- a/config.py +++ b/config.py @@ -15,7 +15,6 @@ from utils.key import KEY_DIR from utils.key import get_key from utils.key import get_secret_key from utils.media import MediaCache -from utils.object_service import ObjectService class ThemeStyle(Enum): @@ -148,6 +147,3 @@ ME = { }, "publicKey": KEY.to_dict(), } - - -OBJECT_SERVICE = ACTOR_SERVICE = ObjectService() diff --git a/utils/object_service.py b/utils/object_service.py deleted file mode 100644 index e46f9b1..0000000 --- a/utils/object_service.py +++ /dev/null @@ -1,21 +0,0 @@ -import logging - -from little_boxes.activitypub import get_backend - -logger = logging.getLogger(__name__) - - -class ObjectService(object): - def __init__(self): - logger.debug("Initializing ObjectService") - self._cache = {} - - def get(self, iri, reload_cache=False): - logger.info(f"get actor {iri} (reload_cache={reload_cache})") - - if not reload_cache and iri in self._cache: - return self._cache[iri] - - obj = get_backend().fetch_iri(iri) - self._cache[iri] = obj - return obj From 4d565a5e4426ccd4cbdb1ef030aafbef2569acf9 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 11 Jul 2018 19:41:52 +0200 Subject: [PATCH 0203/1425] Upgrade to Python 3.7 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8a7a764..8202116 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6 +FROM python:3 ADD . /app WORKDIR /app RUN pip install -r requirements.txt From 8acc17da8bf505b87cbd9e582cd2e37b85135999 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 11 Jul 2018 19:43:10 +0200 Subject: [PATCH 0204/1425] CSS tweaks --- templates/utils.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/utils.html b/templates/utils.html index 2ba93ce..b6b0e32 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -8,7 +8,7 @@
    {{ follower.name or follower.preferredUsername }}
    -@{{ follower.preferredUsername }}@{{ follower.url | get_url | domain }} +@{{ follower.preferredUsername }}@{{ follower.url | get_url | domain }}
    {%- endmacro %} From 1adeee308fb892131d6401108b451d0c020be5c2 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 11 Jul 2018 20:04:48 +0200 Subject: [PATCH 0205/1425] Cache actors --- activitypub.py | 30 +++++++++++++++++++++++++++++- requirements.txt | 1 + 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/activitypub.py b/activitypub.py index 336ede5..1a6a9fe 100644 --- a/activitypub.py +++ b/activitypub.py @@ -10,6 +10,7 @@ from typing import Optional from bson.objectid import ObjectId from feedgen.feed import FeedGenerator from html2text import html2text +from cachetools import LRUCache import tasks from config import BASE_URL @@ -31,6 +32,9 @@ from utils.media import Kind logger = logging.getLogger(__name__) +ACTORS_CACHE = LRUCache(maxsize=256) + + def _remove_id(doc: ap.ObjectType) -> ap.ObjectType: """Helper for removing MongoDB's `_id` field.""" doc = doc.copy() @@ -140,7 +144,7 @@ class MicroblogPubBackend(Backend): ) ) - def fetch_iri(self, iri: str) -> ap.ObjectType: + def _fetch_iri(self, iri: str) -> ap.ObjectType: if iri == ME["id"]: return ME @@ -168,6 +172,30 @@ class MicroblogPubBackend(Backend): # Fetch the URL via HTTP return super().fetch_iri(iri) + def fetch_iri(self, iri: str) -> ap.ObjectType: + if iri == ME["id"]: + return ME + + if iri in ACTORS_CACHE: + return ACTORS_CACHE[iri] + + data = DB.actors.find_one({"remote_id": iri}) + if data: + ACTORS_CACHE[iri] = data['data'] + return data['data'] + + data = self._fetch_iri(iri) + if ap._has_type(data["type"], ap.ACTOR_TYPES): + # Cache the actor + DB.actors.update_one( + {"remote_id": iri}, + {"$set": {"remote_id": iri, "data": data}}, + upsert=True, + ) + ACTORS_CACHE[iri] = data + + return data + @ensure_it_is_me def inbox_check_duplicate(self, as_actor: ap.Person, iri: str) -> bool: return bool(DB.activities.find_one({"box": Box.INBOX.value, "remote_id": iri})) diff --git a/requirements.txt b/requirements.txt index 21ce34c..d2eec2d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,3 +20,4 @@ git+https://github.com/erikriver/opengraph.git git+https://github.com/tsileo/little-boxes.git pyyaml pillow +cachetools From d6d20a972e0f80a63fd271a0a04884e1d934cb40 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 11 Jul 2018 20:27:16 +0200 Subject: [PATCH 0206/1425] Bugfix following page --- activitypub.py | 6 +++--- app.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/activitypub.py b/activitypub.py index 1a6a9fe..aeac7e9 100644 --- a/activitypub.py +++ b/activitypub.py @@ -10,9 +10,9 @@ from typing import Optional from bson.objectid import ObjectId from feedgen.feed import FeedGenerator from html2text import html2text -from cachetools import LRUCache import tasks +from cachetools import LRUCache from config import BASE_URL from config import DB from config import EXTRA_INBOXES @@ -181,8 +181,8 @@ class MicroblogPubBackend(Backend): data = DB.actors.find_one({"remote_id": iri}) if data: - ACTORS_CACHE[iri] = data['data'] - return data['data'] + ACTORS_CACHE[iri] = data["data"] + return data["data"] data = self._fetch_iri(iri) if ap._has_type(data["type"], ap.ACTOR_TYPES): diff --git a/app.py b/app.py index e4f4995..0287efc 100644 --- a/app.py +++ b/app.py @@ -1569,7 +1569,7 @@ def following(): ) following, older_than, newer_than = paginated_query(DB.activities, q) - following = [get_backend.fetch_iri(doc["activity"]["object"]) for doc in following] + following = [get_backend().fetch_iri(doc["activity"]["object"]) for doc in following] return render_template( "following.html", following_data=following, From a36b56af4ebdf0899cb1d58b8325c5695541cfc7 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 11 Jul 2018 21:53:47 +0200 Subject: [PATCH 0207/1425] Move the caching to celery --- activitypub.py | 11 ++-------- tasks.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 9 deletions(-) diff --git a/activitypub.py b/activitypub.py index aeac7e9..f92769d 100644 --- a/activitypub.py +++ b/activitypub.py @@ -18,7 +18,6 @@ from config import DB from config import EXTRA_INBOXES from config import ID from config import ME -from config import MEDIA_CACHE from config import USER_AGENT from config import USERNAME from little_boxes import activitypub as ap @@ -27,7 +26,6 @@ from little_boxes.activitypub import _to_list from little_boxes.backend import Backend from little_boxes.errors import ActivityGoneError from little_boxes.errors import Error -from utils.media import Kind logger = logging.getLogger(__name__) @@ -97,13 +95,8 @@ class MicroblogPubBackend(Backend): } ) - # Generates thumbnails for the actor's icon and the attachments if any - actor = activity.get_actor() - if actor.icon: - MEDIA_CACHE.cache(actor.icon["url"], Kind.ACTOR_ICON) - if activity.type == ap.ActivityType.CREATE.value: - for attachment in activity.get_object()._data.get("attachment", []): - MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT) + tasks.process_new_activity.delay(activity.id) + tasks.cache_attachments(activity.id) @ensure_it_is_me def outbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: diff --git a/tasks.py b/tasks.py index a5c85db..79d2857 100644 --- a/tasks.py +++ b/tasks.py @@ -7,6 +7,7 @@ import requests from celery import Celery from requests.exceptions import HTTPError +from little_boxes import activitypub as ap from config import DB from config import HEADERS from config import KEY @@ -14,6 +15,8 @@ from config import USER_AGENT from little_boxes.httpsig import HTTPSigAuth from little_boxes.linked_data_sig import generate_signature from utils.opengraph import fetch_og_metadata +from utils.media import Kind +from config import MEDIA_CACHE log = logging.getLogger(__name__) app = Celery( @@ -22,6 +25,58 @@ app = Celery( SigAuth = HTTPSigAuth(KEY) +@app.task(bind=True, max_retries=12) +def process_new_activity(self, iri: str) -> None: + try: + activity = ap.fetch_remote_activity(iri) + log.info(f"activity={activity!r}") + + tag_stream = False + if activity.has_type(ap.ActivityType.ANNOUCE): + tag_stream = True + elif activity.has_type(ap.ActivityType.CREATE): + note = activity.get_object() + if not note.inReplyTo: + tag_stream = True + + log.info(f"{iri} tag_stream={tag_stream}") + DB.update_one({"remote_id": activity.id}, {"$set": {"meta.stream": tag_stream}}) + + log.info(f"new activity {iri} processed") + except Exception as err: + log.exception(f"failed to process new activity {iri}") + self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) + + +@app.task(bind=True, max_retries=12) +def cache_attachments(self, iri: str) -> None: + try: + activity = ap.fetch_remote_activity(iri) + log.info(f"activity={activity!r}") + # Generates thumbnails for the actor's icon and the attachments if any + + actor = activity.get_actor() + + # Update the cached actor + DB.actors.update_one( + {"remote_id": iri}, + {"$set": {"remote_id": iri, "data": actor.to_dict(embed=True)}}, + upsert=True, + ) + + if actor.icon: + MEDIA_CACHE.cache(actor.icon["url"], Kind.ACTOR_ICON) + if activity.has_type(ap.ActivityType.CREATE): + for attachment in activity.get_object()._data.get("attachment", []): + MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT) + + log.info(f"attachmwents cached for {iri}") + + except Exception as err: + log.exception(f"failed to process new activity {iri}") + self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) + + @app.task(bind=True, max_retries=12) def post_to_inbox(self, payload: str, to: str) -> None: try: From a7bc6fa1a2776ecf7e07320b68455a7a8e20e22c Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 11 Jul 2018 22:00:18 +0200 Subject: [PATCH 0208/1425] Tweak the tasks --- activitypub.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/activitypub.py b/activitypub.py index f92769d..6c6372d 100644 --- a/activitypub.py +++ b/activitypub.py @@ -95,8 +95,9 @@ class MicroblogPubBackend(Backend): } ) - tasks.process_new_activity.delay(activity.id) tasks.cache_attachments(activity.id) + if box == Box.INBOX: + tasks.process_new_activity.delay(activity.id) @ensure_it_is_me def outbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: From fa07450da324f7399873dc75a0fbf52cb49b45f8 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 11 Jul 2018 22:52:20 +0200 Subject: [PATCH 0209/1425] Fix the build --- activitypub.py | 2 +- tasks.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/activitypub.py b/activitypub.py index 6c6372d..8deed50 100644 --- a/activitypub.py +++ b/activitypub.py @@ -95,7 +95,7 @@ class MicroblogPubBackend(Backend): } ) - tasks.cache_attachments(activity.id) + tasks.cache_attachments.delay(activity.id) if box == Box.INBOX: tasks.process_new_activity.delay(activity.id) diff --git a/tasks.py b/tasks.py index 79d2857..2787a0b 100644 --- a/tasks.py +++ b/tasks.py @@ -66,6 +66,7 @@ def cache_attachments(self, iri: str) -> None: if actor.icon: MEDIA_CACHE.cache(actor.icon["url"], Kind.ACTOR_ICON) + if activity.has_type(ap.ActivityType.CREATE): for attachment in activity.get_object()._data.get("attachment", []): MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT) From 63ca0b68e1823f35d5ec6b925f630308bd8df56c Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 11 Jul 2018 23:22:47 +0200 Subject: [PATCH 0210/1425] Fix the tasks and formatting --- activitypub.py | 27 ++++++++++++----------- app.py | 46 +++++++++++++++++++++++++++------------- config.py | 2 +- tasks.py | 41 +++++++++++++---------------------- tests/federation_test.py | 1 - utils/opengraph.py | 1 - 6 files changed, 62 insertions(+), 56 deletions(-) diff --git a/activitypub.py b/activitypub.py index 8deed50..2e748fa 100644 --- a/activitypub.py +++ b/activitypub.py @@ -8,11 +8,16 @@ from typing import List from typing import Optional from bson.objectid import ObjectId +from cachetools import LRUCache from feedgen.feed import FeedGenerator from html2text import html2text +from little_boxes import activitypub as ap +from little_boxes import strtobool +from little_boxes.activitypub import _to_list +from little_boxes.backend import Backend +from little_boxes.errors import ActivityGoneError +from little_boxes.errors import Error -import tasks -from cachetools import LRUCache from config import BASE_URL from config import DB from config import EXTRA_INBOXES @@ -20,12 +25,6 @@ from config import ID from config import ME from config import USER_AGENT from config import USERNAME -from little_boxes import activitypub as ap -from little_boxes import strtobool -from little_boxes.activitypub import _to_list -from little_boxes.backend import Backend -from little_boxes.errors import ActivityGoneError -from little_boxes.errors import Error logger = logging.getLogger(__name__) @@ -95,9 +94,10 @@ class MicroblogPubBackend(Backend): } ) - tasks.cache_attachments.delay(activity.id) - if box == Box.INBOX: - tasks.process_new_activity.delay(activity.id) + self.save_cb(box, activity.id) + + def set_save_cb(self, cb): + self.save_cb = cb @ensure_it_is_me def outbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: @@ -198,9 +198,12 @@ class MicroblogPubBackend(Backend): def inbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: self.save(Box.INBOX, activity) + def set_post_to_remote_inbox(self, cb): + self.post_to_remote_inbox_cb = cb + @ensure_it_is_me def post_to_remote_inbox(self, as_actor: ap.Person, payload: str, to: str) -> None: - tasks.post_to_inbox.delay(payload, to) + self.post_to_remote_inbox_cb(payload, to) @ensure_it_is_me def new_follower(self, as_actor: ap.Person, follow: ap.Follow) -> None: diff --git a/app.py b/app.py index 0287efc..425b5b6 100644 --- a/app.py +++ b/app.py @@ -33,12 +33,27 @@ from flask import url_for from flask_wtf.csrf import CSRFProtect from html2text import html2text from itsdangerous import BadSignature +from little_boxes import activitypub as ap +from little_boxes.activitypub import ActivityType +from little_boxes.activitypub import _to_list +from little_boxes.activitypub import clean_activity +from little_boxes.activitypub import get_backend +from little_boxes.content_helper import parse_markdown +from little_boxes.errors import ActivityGoneError +from little_boxes.errors import ActivityNotFoundError +from little_boxes.errors import Error +from little_boxes.errors import NotFromOutboxError +from little_boxes.httpsig import HTTPSigAuth +from little_boxes.httpsig import verify_request +from little_boxes.webfinger import get_actor_url +from little_boxes.webfinger import get_remote_follow_template from passlib.hash import bcrypt from u2flib_server import u2f from werkzeug.utils import secure_filename import activitypub import config +import tasks from activitypub import Box from activitypub import embed_collection from config import ADMIN_API_KEY @@ -58,24 +73,23 @@ from config import USERNAME from config import VERSION from config import _drop_db from config import custom_cache_purge_hook -from little_boxes import activitypub as ap -from little_boxes.activitypub import ActivityType -from little_boxes.activitypub import _to_list -from little_boxes.activitypub import clean_activity -from little_boxes.activitypub import get_backend -from little_boxes.content_helper import parse_markdown -from little_boxes.errors import ActivityGoneError -from little_boxes.errors import ActivityNotFoundError -from little_boxes.errors import Error -from little_boxes.errors import NotFromOutboxError -from little_boxes.httpsig import HTTPSigAuth -from little_boxes.httpsig import verify_request -from little_boxes.webfinger import get_actor_url -from little_boxes.webfinger import get_remote_follow_template from utils.key import get_secret_key from utils.media import Kind back = activitypub.MicroblogPubBackend() + + +def save_cb(box: Box, iri: str) -> None: + tasks.cache_attachments.delay(iri) + if box == Box.INBOX: + tasks.process_new_activity.delay(iri) + + +back.set_save_cb(save_cb) + + +back.set_post_to_remote_inbox(tasks.post_to_inbox.delay) + ap.use_backend(back) MY_PERSON = ap.Person(**ME) @@ -1569,7 +1583,9 @@ def following(): ) following, older_than, newer_than = paginated_query(DB.activities, q) - following = [get_backend().fetch_iri(doc["activity"]["object"]) for doc in following] + following = [ + get_backend().fetch_iri(doc["activity"]["object"]) for doc in following + ] return render_template( "following.html", following_data=following, diff --git a/config.py b/config.py index 9e7b844..8c78fb6 100644 --- a/config.py +++ b/config.py @@ -8,9 +8,9 @@ import requests import sass import yaml from itsdangerous import JSONWebSignatureSerializer +from little_boxes import strtobool from pymongo import MongoClient -from little_boxes import strtobool from utils.key import KEY_DIR from utils.key import get_key from utils.key import get_secret_key diff --git a/tasks.py b/tasks.py index 2787a0b..7f3a3a8 100644 --- a/tasks.py +++ b/tasks.py @@ -5,18 +5,18 @@ import random import requests from celery import Celery +from little_boxes import activitypub as ap +from little_boxes.httpsig import HTTPSigAuth +from little_boxes.linked_data_sig import generate_signature from requests.exceptions import HTTPError -from little_boxes import activitypub as ap +import activitypub from config import DB from config import HEADERS from config import KEY -from config import USER_AGENT -from little_boxes.httpsig import HTTPSigAuth -from little_boxes.linked_data_sig import generate_signature -from utils.opengraph import fetch_og_metadata -from utils.media import Kind from config import MEDIA_CACHE +from config import USER_AGENT +from utils.media import Kind log = logging.getLogger(__name__) app = Celery( @@ -25,6 +25,10 @@ app = Celery( SigAuth = HTTPSigAuth(KEY) +back = activitypub.MicroblogPubBackend() +ap.use_backend(back) + + @app.task(bind=True, max_retries=12) def process_new_activity(self, iri: str) -> None: try: @@ -32,7 +36,7 @@ def process_new_activity(self, iri: str) -> None: log.info(f"activity={activity!r}") tag_stream = False - if activity.has_type(ap.ActivityType.ANNOUCE): + if activity.has_type(ap.ActivityType.ANNOUNCE): tag_stream = True elif activity.has_type(ap.ActivityType.CREATE): note = activity.get_object() @@ -40,7 +44,9 @@ def process_new_activity(self, iri: str) -> None: tag_stream = True log.info(f"{iri} tag_stream={tag_stream}") - DB.update_one({"remote_id": activity.id}, {"$set": {"meta.stream": tag_stream}}) + DB.activities.update_one( + {"remote_id": activity.id}, {"$set": {"meta.stream": tag_stream}} + ) log.info(f"new activity {iri} processed") except Exception as err: @@ -71,7 +77,7 @@ def cache_attachments(self, iri: str) -> None: for attachment in activity.get_object()._data.get("attachment", []): MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT) - log.info(f"attachmwents cached for {iri}") + log.info(f"attachments cached for {iri}") except Exception as err: log.exception(f"failed to process new activity {iri}") @@ -105,20 +111,3 @@ def post_to_inbox(self, payload: str, to: str) -> None: log.info("client error, no retry") return self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) - - -@app.task(bind=True, max_retries=12) -def fetch_og(self, col, remote_id): - try: - log.info("fetch_og_meta remote_id=%s col=%s", remote_id, col) - if col == "INBOX": - log.info( - "%d links saved", fetch_og_metadata(USER_AGENT, DB.inbox, remote_id) - ) - elif col == "OUTBOX": - log.info( - "%d links saved", fetch_og_metadata(USER_AGENT, DB.outbox, remote_id) - ) - except Exception as err: - self.log.exception("failed") - self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) diff --git a/tests/federation_test.py b/tests/federation_test.py index f926812..20569bb 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -5,7 +5,6 @@ from typing import Tuple import requests from html2text import html2text - from little_boxes.collection import parse_collection diff --git a/utils/opengraph.py b/utils/opengraph.py index 597ad3c..b543269 100644 --- a/utils/opengraph.py +++ b/utils/opengraph.py @@ -1,7 +1,6 @@ import opengraph import requests from bs4 import BeautifulSoup - from little_boxes.urlutils import check_url from little_boxes.urlutils import is_url_valid From 6b464d2a6dd2a2ae39bb27dfd53d34fe9e4cee06 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 11 Jul 2018 23:49:43 +0200 Subject: [PATCH 0211/1425] Fix thread issue --- app.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 425b5b6..8229ea6 100644 --- a/app.py +++ b/app.py @@ -763,8 +763,10 @@ def _build_thread(data, include_children=True): if rep_id == root_id: continue reply_of = rep["activity"]["object"]["inReplyTo"] - idx[reply_of]["_nodes"].append(rep) - + try: + idx[reply_of]["_nodes"].append(rep) + except: + pass # Flatten the tree thread = [] From ff55b9e9dca980e629ab1b37f063c5e7d2e65677 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 12 Jul 2018 08:05:23 +0200 Subject: [PATCH 0212/1425] Fix thread issue --- app.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/app.py b/app.py index 8229ea6..0febe0f 100644 --- a/app.py +++ b/app.py @@ -737,7 +737,8 @@ def _build_thread(data, include_children=True): print(data) root_id = data["meta"].get("thread_root_parent", data["activity"]["object"]["id"]) - query = {"$or": [{"meta.thread_root_parent": root_id, "type": "Create"}]} + query = {"$or": [{"meta.thread_root_parent": root_id, "type": "Create"}, + {"activity.object.id": root_id}]} if data["activity"]["object"].get("inReplyTo"): query["$or"].append( {"activity.object.id": data["activity"]["object"]["inReplyTo"]} @@ -763,10 +764,8 @@ def _build_thread(data, include_children=True): if rep_id == root_id: continue reply_of = rep["activity"]["object"]["inReplyTo"] - try: - idx[reply_of]["_nodes"].append(rep) - except: - pass + idx[reply_of]["_nodes"].append(rep) + # Flatten the tree thread = [] From 4a882fb7bad498162b2a0e0a154d5b4adebe8932 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 13 Jul 2018 00:44:33 +0200 Subject: [PATCH 0213/1425] Tweak the Celery tasks retry --- tasks.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tasks.py b/tasks.py index 7f3a3a8..9206702 100644 --- a/tasks.py +++ b/tasks.py @@ -6,6 +6,8 @@ import random import requests from celery import Celery from little_boxes import activitypub as ap +from little_boxes.errors import ActivityGoneError +from little_boxes.errors import ActivityNotFoundError from little_boxes.httpsig import HTTPSigAuth from little_boxes.linked_data_sig import generate_signature from requests.exceptions import HTTPError @@ -49,6 +51,8 @@ def process_new_activity(self, iri: str) -> None: ) log.info(f"new activity {iri} processed") + except (ActivityGoneError, ActivityNotFoundError): + log.exception("failed to process activity {iri}") except Exception as err: log.exception(f"failed to process new activity {iri}") self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) @@ -79,6 +83,8 @@ def cache_attachments(self, iri: str) -> None: log.info(f"attachments cached for {iri}") + except (ActivityGoneError, ActivityNotFoundError): + log.exception("failed to process activity {iri}") except Exception as err: log.exception(f"failed to process new activity {iri}") self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) From 2a9de02ecf48ea88ac5497aa54af8d4121e88165 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 13 Jul 2018 00:45:29 +0200 Subject: [PATCH 0214/1425] Update the Makefile --- Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 846d967..a387a2b 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,5 @@ PYTHON=python -css: - $(PYTHON) -c "import sass; sass.compile(dirname=('sass', 'static/css'), output_style='compressed')" - password: $(PYTHON) -c "import bcrypt; from getpass import getpass; print(bcrypt.hashpw(getpass().encode('utf-8'), bcrypt.gensalt()).decode('utf-8'))" @@ -11,6 +8,7 @@ docker: docker build . -t microblogpub:latest reload-fed: + docker build . -t microblogpub:latest docker-compose -p instance2 -f docker-compose-tests.yml stop docker-compose -p instance1 -f docker-compose-tests.yml stop WEB_PORT=5006 CONFIG_DIR=./tests/fixtures/instance1/config docker-compose -p instance1 -f docker-compose-tests.yml up -d --force-recreate --build From feb611f4ce54ca563fead48e3b04add06a2aa5a7 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 13 Jul 2018 01:03:10 +0200 Subject: [PATCH 0215/1425] Fix the tasks logging --- tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks.py b/tasks.py index 9206702..bbb8796 100644 --- a/tasks.py +++ b/tasks.py @@ -52,7 +52,7 @@ def process_new_activity(self, iri: str) -> None: log.info(f"new activity {iri} processed") except (ActivityGoneError, ActivityNotFoundError): - log.exception("failed to process activity {iri}") + log.exception(f"dropping activity {iri}") except Exception as err: log.exception(f"failed to process new activity {iri}") self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) @@ -84,7 +84,7 @@ def cache_attachments(self, iri: str) -> None: log.info(f"attachments cached for {iri}") except (ActivityGoneError, ActivityNotFoundError): - log.exception("failed to process activity {iri}") + log.exception(f"dropping activity {iri}") except Exception as err: log.exception(f"failed to process new activity {iri}") self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) From d8f7967e6a706547078215a99f005d12553565ca Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 14 Jul 2018 12:14:08 +0200 Subject: [PATCH 0216/1425] Tweak the admin UI --- Makefile | 4 ++++ activitypub.py | 30 ++++++++++++++++++------------ app.py | 2 +- tasks.py | 3 +++ templates/following.html | 13 ++++++++++++- templates/utils.html | 15 ++++++++------- 6 files changed, 46 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index a387a2b..6c7a767 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,10 @@ reload-fed: WEB_PORT=5006 CONFIG_DIR=./tests/fixtures/instance1/config docker-compose -p instance1 -f docker-compose-tests.yml up -d --force-recreate --build WEB_PORT=5007 CONFIG_DIR=./tests/fixtures/instance2/config docker-compose -p instance2 -f docker-compose-tests.yml up -d --force-recreate --build +reload-dev: + docker build . -t microblogpub:latest + docker-compose -f docker-compose-dev.yml up -d --force-recreate + update: git pull docker build . -t microblogpub:latest diff --git a/activitypub.py b/activitypub.py index 2e748fa..6ce6483 100644 --- a/activitypub.py +++ b/activitypub.py @@ -103,25 +103,31 @@ class MicroblogPubBackend(Backend): def outbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: self.save(Box.OUTBOX, activity) + def followers(self) -> List[str]: + q = { + "box": Box.INBOX.value, + "type": ap.ActivityType.FOLLOW.value, + "meta.undo": False, + } + return [doc["activity"]["actor"] for doc in DB.activities.find(q)] + + def following(self) -> List[str]: + q = { + "box": Box.OUTBOX.value, + "type": ap.ActivityType.FOLLOW.value, + "meta.undo": False, + } + return [doc["activity"]["object"] for doc in DB.activities.find(q)] + def parse_collection( self, payload: Optional[Dict[str, Any]] = None, url: Optional[str] = None ) -> List[str]: """Resolve/fetch a `Collection`/`OrderedCollection`.""" # Resolve internal collections via MongoDB directly if url == ID + "/followers": - q = { - "box": Box.INBOX.value, - "type": ap.ActivityType.FOLLOW.value, - "meta.undo": False, - } - return [doc["activity"]["actor"] for doc in DB.activities.find(q)] + return self.followers() elif url == ID + "/following": - q = { - "box": Box.OUTBOX.value, - "type": ap.ActivityType.FOLLOW.value, - "meta.undo": False, - } - return [doc["activity"]["object"] for doc in DB.activities.find(q)] + return self.following() return super().parse_collection(payload, url) diff --git a/app.py b/app.py index 0febe0f..d431bc1 100644 --- a/app.py +++ b/app.py @@ -1585,7 +1585,7 @@ def following(): following, older_than, newer_than = paginated_query(DB.activities, q) following = [ - get_backend().fetch_iri(doc["activity"]["object"]) for doc in following + (doc["remote_id"], get_backend().fetch_iri(doc["activity"]["object"])) for doc in following ] return render_template( "following.html", diff --git a/tasks.py b/tasks.py index bbb8796..79a1d51 100644 --- a/tasks.py +++ b/tasks.py @@ -37,6 +37,9 @@ def process_new_activity(self, iri: str) -> None: activity = ap.fetch_remote_activity(iri) log.info(f"activity={activity!r}") + # Is the activity expected? + # following = ap.get_backend().following() + tag_stream = False if activity.has_type(ap.ActivityType.ANNOUNCE): tag_stream = True diff --git a/templates/following.html b/templates/following.html index c249b7d..1b96ed1 100644 --- a/templates/following.html +++ b/templates/following.html @@ -8,7 +8,18 @@ {% include "header.html" %}
    - {% for followed in following_data %} + {% for (follow_id, followed) in following_data %} + {% if session.logged_in %} +
    +
    + + + + +
    +
    + + {% endif %}
    {{ utils.display_actor_inline(followed, size=80) }}
    diff --git a/templates/utils.html b/templates/utils.html index b6b0e32..d0858b1 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -65,18 +65,19 @@ permalink {% if meta.count_reply %}{{ meta.count_reply }} replies{% endif %} -{% if meta.count_boost %}{{ meta.count_boost }} boosts{% endif %} -{% if meta.count_like %}{{ meta.count_like }} likes{% endif %} +{% if meta.count_boost and obj.id | is_from_outbox %}{{ meta.count_boost }} boosts{% endif %} +{% if meta.count_like and obj.id | is_from_outbox %}{{ meta.count_like }} likes{% endif %} +{% endif %} + +{% if session.logged_in %} +{% set perma_id = obj.id | permalink_id %} +{% set redir = request.path + "#activity-" + perma_id %} +{% set aid = obj.id | quote_plus %} {% endif %} {% if ui and session.logged_in %} -{% set aid = obj.id | quote_plus %} reply - -{% set perma_id = obj.id | permalink_id %} -{% set redir = request.path + "#activity-" + perma_id %} - {% if meta.boosted %}
    From 0a53cba5f57fa31abcd3b4b5da76f1859ac92c41 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 14 Jul 2018 12:29:46 +0200 Subject: [PATCH 0217/1425] Display threads locally when possible --- app.py | 17 +++++++++++++++++ templates/utils.html | 17 ++++++++++------- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/app.py b/app.py index d431bc1..ec77748 100644 --- a/app.py +++ b/app.py @@ -1168,6 +1168,23 @@ def admin(): ) +@app.route("/admin/thread") +@login_required +def admin_thread(): + data = DB.activities.find_one( + {"$or": [{"remote_id": request.args.get("oid")}, {"activity.object.id": request.args.get("oid")}]} + ) + if not data: + abort(404) + if data["meta"].get("deleted", False): + abort(410) + thread = _build_thread(data) + + return render_template( + "note.html", thread=thread, note=data + ) + + @app.route("/admin/new", methods=["GET"]) @login_required def admin_new(): diff --git a/templates/utils.html b/templates/utils.html index d0858b1..f2893d1 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -64,18 +64,21 @@ {% else %} permalink -{% if meta.count_reply %}{{ meta.count_reply }} replies{% endif %} -{% if meta.count_boost and obj.id | is_from_outbox %}{{ meta.count_boost }} boosts{% endif %} -{% if meta.count_like and obj.id | is_from_outbox %}{{ meta.count_like }} likes{% endif %} -{% endif %} - {% if session.logged_in %} {% set perma_id = obj.id | permalink_id %} {% set redir = request.path + "#activity-" + perma_id %} {% set aid = obj.id | quote_plus %} {% endif %} -{% if ui and session.logged_in %} +{% if meta.count_reply and obj.id | is_from_outbox %}{{ meta.count_reply }} replies +{% elif meta.count_reply and session.logged_in %} +{{ meta.count_reply }} replies{% endif %} + +{% if meta.count_boost and obj.id | is_from_outbox %}{{ meta.count_boost }} boosts{% endif %} +{% if meta.count_like and obj.id | is_from_outbox %}{{ meta.count_like }} likes{% endif %} + +{% if session.logged_in %} +{% if ui%} reply {% if meta.boosted %} @@ -112,7 +115,6 @@ {% endif %} -{% if session.logged_in %} {% if obj.id | is_from_outbox %} @@ -130,6 +132,7 @@ {% endif %} {% endif %} +{% endif %}
    From 3497c09035e064ad9275a0708d7282e69d274136 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 14 Jul 2018 12:54:24 +0200 Subject: [PATCH 0218/1425] Tweak the admin stream --- app.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app.py b/app.py index ec77748..478a9f2 100644 --- a/app.py +++ b/app.py @@ -1356,8 +1356,7 @@ def api_undo(): @login_required def admin_stream(): q = { - "box": Box.INBOX.value, - "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, + "meta.stream": True, "meta.deleted": False, } inbox_data, older_than, newer_than = paginated_query(DB.activities, q) From a9379a520585a34b0ef4b8cb1c681f1c5f5b1d0a Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 14 Jul 2018 13:19:30 +0200 Subject: [PATCH 0219/1425] Tweak the logging --- app.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/app.py b/app.py index 478a9f2..9dbea90 100644 --- a/app.py +++ b/app.py @@ -737,8 +737,12 @@ def _build_thread(data, include_children=True): print(data) root_id = data["meta"].get("thread_root_parent", data["activity"]["object"]["id"]) - query = {"$or": [{"meta.thread_root_parent": root_id, "type": "Create"}, - {"activity.object.id": root_id}]} + query = { + "$or": [ + {"meta.thread_root_parent": root_id, "type": "Create"}, + {"activity.object.id": root_id}, + ] + } if data["activity"]["object"].get("inReplyTo"): query["$or"].append( {"activity.object.id": data["activity"]["object"]["inReplyTo"]} @@ -1172,7 +1176,12 @@ def admin(): @login_required def admin_thread(): data = DB.activities.find_one( - {"$or": [{"remote_id": request.args.get("oid")}, {"activity.object.id": request.args.get("oid")}]} + { + "$or": [ + {"remote_id": request.args.get("oid")}, + {"activity.object.id": request.args.get("oid")}, + ] + } ) if not data: abort(404) @@ -1180,9 +1189,7 @@ def admin_thread(): abort(410) thread = _build_thread(data) - return render_template( - "note.html", thread=thread, note=data - ) + return render_template("note.html", thread=thread, note=data) @app.route("/admin/new", methods=["GET"]) @@ -1269,15 +1276,18 @@ def _user_api_arg(key: str, **kwargs): if not oid: if "default" in kwargs: + app.logger.info(f'{key}={kwargs.get("default")}') return kwargs.get("default") raise ValueError(f"missing {key}") + app.logger.info(f"{key}={oid}") return oid def _user_api_get_note(from_outbox: bool = False): oid = _user_api_arg("id") + app.logger.info(f"fetching {oid}") note = ap.parse_activity(get_backend().fetch_iri(oid), expected=ActivityType.NOTE) if from_outbox and not note.id.startswith(ID): raise NotFromOutboxError( @@ -1355,10 +1365,7 @@ def api_undo(): @app.route("/admin/stream") @login_required def admin_stream(): - q = { - "meta.stream": True, - "meta.deleted": False, - } + q = {"meta.stream": True, "meta.deleted": False} inbox_data, older_than, newer_than = paginated_query(DB.activities, q) return render_template( @@ -1601,7 +1608,8 @@ def following(): following, older_than, newer_than = paginated_query(DB.activities, q) following = [ - (doc["remote_id"], get_backend().fetch_iri(doc["activity"]["object"])) for doc in following + (doc["remote_id"], get_backend().fetch_iri(doc["activity"]["object"])) + for doc in following ] return render_template( "following.html", From 364c1d9273fb0d3cd83e97cbaa1771c77561b076 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 14 Jul 2018 13:45:06 +0200 Subject: [PATCH 0220/1425] More logging --- activitypub.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/activitypub.py b/activitypub.py index 6ce6483..7391733 100644 --- a/activitypub.py +++ b/activitypub.py @@ -170,6 +170,7 @@ class MicroblogPubBackend(Backend): return data["activity"] # Fetch the URL via HTTP + logger.info(f"dereference {iri} via HTTP") return super().fetch_iri(iri) def fetch_iri(self, iri: str) -> ap.ObjectType: @@ -177,14 +178,17 @@ class MicroblogPubBackend(Backend): return ME if iri in ACTORS_CACHE: + logger.info(f"{iri} found in cache") return ACTORS_CACHE[iri] data = DB.actors.find_one({"remote_id": iri}) if data: + logger.info(f"{iri} found in DB cache") ACTORS_CACHE[iri] = data["data"] return data["data"] data = self._fetch_iri(iri) + logger.debug(f"_fetch_iri({iri!r}) == {data!r}") if ap._has_type(data["type"], ap.ACTOR_TYPES): # Cache the actor DB.actors.update_one( From e3284416d2d80b239fab7a825b4fbaa5c93f2007 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 15 Jul 2018 21:12:57 +0200 Subject: [PATCH 0221/1425] Better Link support --- app.py | 4 +++- templates/index.html | 4 ++-- templates/stream.html | 2 +- templates/utils.html | 17 +++++++++-------- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/app.py b/app.py index 9dbea90..91003bc 100644 --- a/app.py +++ b/app.py @@ -269,8 +269,10 @@ def domain(url): def get_url(u): if isinstance(u, dict): return u["href"] - else: + elif isinstance(u, str): return u + else: + raise ValueError(f"unexpected URL field type: {type(u)}: {u!r}") @app.template_filter() diff --git a/templates/index.html b/templates/index.html index 6cc2e37..a398087 100644 --- a/templates/index.html +++ b/templates/index.html @@ -27,7 +27,7 @@ {% set boost_actor = item.activity.actor | get_actor %} {% if session.logged_in %}
    - {{ boost_actor.name }} boosted + {{ boost_actor.name }} boosted @@ -37,7 +37,7 @@
    {% else %}

    - {{ boost_actor.name }} boosted + {{ boost_actor.name }} boosted

    {% endif %} {% if item.meta.object %} diff --git a/templates/stream.html b/templates/stream.html index 26fdc3b..365288a 100644 --- a/templates/stream.html +++ b/templates/stream.html @@ -14,7 +14,7 @@ {% if item | has_type('Announce') %} {% set boost_actor = item.activity.actor | get_actor %} -

    {{ boost_actor.name or boost_actor.preferredUsername }} boosted

    +

    {{ boost_actor.name or boost_actor.preferredUsername }} boosted

    {% if item.meta.object %} {{ utils.display_note(item.meta.object, ui=True) }} {% endif %} diff --git a/templates/utils.html b/templates/utils.html index f2893d1..ffb4c11 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -31,7 +31,7 @@ {% if not perma %} - + {% endif %} @@ -47,7 +47,7 @@
      {% endif %} {% for a in obj.attachment %} - {% if a.url | is_img %} + {% if a.url | get_url | is_img %} {% else %}
    • {% if a.filename %}{{ a.filename }}{% else %}{{ a.url }}{% endif %}
      • @@ -61,8 +61,10 @@
      {% if perma %}{{ obj.published | format_time }} -{% else %} -permalink +{% endif %} +{% if not obj.id | is_from_outbox %} +permalink +{% endif %} {% if session.logged_in %} {% set perma_id = obj.id | permalink_id %} @@ -70,12 +72,12 @@ {% set aid = obj.id | quote_plus %} {% endif %} -{% if meta.count_reply and obj.id | is_from_outbox %}{{ meta.count_reply }} replies +{% if meta.count_reply and obj.id | is_from_outbox %}{{ meta.count_reply }} replies {% elif meta.count_reply and session.logged_in %} {{ meta.count_reply }} replies{% endif %} -{% if meta.count_boost and obj.id | is_from_outbox %}{{ meta.count_boost }} boosts{% endif %} -{% if meta.count_like and obj.id | is_from_outbox %}{{ meta.count_like }} likes{% endif %} +{% if meta.count_boost and obj.id | is_from_outbox %}{{ meta.count_boost }} boosts{% endif %} +{% if meta.count_like and obj.id | is_from_outbox %}{{ meta.count_like }} likes{% endif %} {% if session.logged_in %} {% if ui%} @@ -132,7 +134,6 @@ {% endif %} {% endif %} -{% endif %}
      From 5323995633b318d2f8e5b493ab7f83adff976fb0 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 15 Jul 2018 21:25:09 +0200 Subject: [PATCH 0222/1425] Tweak the login page --- app.py | 3 +++ templates/layout.html | 3 ++- templates/login.html | 15 ++++++++++++--- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/app.py b/app.py index 91003bc..d289858 100644 --- a/app.py +++ b/app.py @@ -471,6 +471,9 @@ def admin_logout(): @app.route("/login", methods=["POST", "GET"]) def admin_login(): + if session.get("logged_in") is True: + return redirect(url_for("admin_notifications")) + devices = [doc["device"] for doc in DB.u2f.find()] u2f_enabled = True if devices else False if request.method == "POST": diff --git a/templates/layout.html b/templates/layout.html index 21522fa..5956de5 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -12,6 +12,7 @@ {% block links %}{% endblock %} {% if config.THEME_COLOR %}{% endif %} +{% block headers %}{% endblock %} {% if logged_in %} @@ -20,7 +21,7 @@
    • New
    • Stream
    • Notifications
      • -
    • Logout
      • +
    • Logout
    {% endif %} diff --git a/templates/login.html b/templates/login.html index 603808c..fde73d2 100644 --- a/templates/login.html +++ b/templates/login.html @@ -1,11 +1,20 @@ {% extends "layout.html" %} {% import 'utils.html' as utils %} {% block title %}Login - {{ config.NAME }}{% endblock %} -{% block header %} +{% block headers %} + {% endblock %} {% block content %} -
    - {% if session.logged_in %}logged{% else%}not logged{%endif%} +
    From 3274289950228e3888178c2f06d283cbf13daf88 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 15 Jul 2018 21:33:59 +0200 Subject: [PATCH 0223/1425] Fix the permalink --- templates/utils.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templates/utils.html b/templates/utils.html index ffb4c11..f78cada 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -61,8 +61,10 @@
    {% if perma %}{{ obj.published | format_time }} +{% if not (obj.id | is_from_outbox) %} +permalink {% endif %} -{% if not obj.id | is_from_outbox %} +{% else %} permalink {% endif %} From 1c778dbd38e16094396242e54411fa26295d1421 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 15 Jul 2018 21:38:15 +0200 Subject: [PATCH 0224/1425] Fix the template filter --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index d289858..834150f 100644 --- a/app.py +++ b/app.py @@ -272,7 +272,7 @@ def get_url(u): elif isinstance(u, str): return u else: - raise ValueError(f"unexpected URL field type: {type(u)}: {u!r}") + return u @app.template_filter() From 11ad9ec10a956a385373fd273a05551ef691b9c3 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 15 Jul 2018 22:20:38 +0200 Subject: [PATCH 0225/1425] Tweak the admin bar --- sass/base_theme.scss | 11 ++++++++--- templates/layout.html | 2 ++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/sass/base_theme.scss b/sass/base_theme.scss index b5dd5c6..dae8140 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -53,13 +53,17 @@ a:hover { background: $primary-color; color: $background-color; } +#admin-menu-wrapper { + padding: 10px; + margin:0 auto; + width: 100%; + background: $color-menu-background; + max-width: 720px; + #admin-menu { list-style-type: none; display: inline; padding: 10px; - width: 720px; - margin:0 auto; - background: $color-menu-background; color: $color-light; border-radius-bottom-left: 2px; border-radius-bottom-right: 2px; @@ -77,6 +81,7 @@ a:hover { } } } +} #header { margin-bottom: 70px; diff --git a/templates/layout.html b/templates/layout.html index 5956de5..42b2e95 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -16,6 +16,7 @@ {% if logged_in %} +
    +
    {% endif %} From f44f9992c9ecd319d643eb4faf01dead1680d717 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 16 Jul 2018 22:24:14 +0200 Subject: [PATCH 0226/1425] Add new debug endpoint --- app.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app.py b/app.py index 834150f..eebdbc6 100644 --- a/app.py +++ b/app.py @@ -566,6 +566,11 @@ def u2f_register(): ####### # Activity pub routes +@app.route("/drop_cache") +@login_required +def drop_cache(): + DB.actors.drop() + return "Done" @app.route("/migration1_step1") From 854f48495e5bdfda39c07a564fae8a7083275093 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 16 Jul 2018 22:41:17 +0200 Subject: [PATCH 0227/1425] Disable actor caching in the DB for debug --- activitypub.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/activitypub.py b/activitypub.py index 7391733..83e58ab 100644 --- a/activitypub.py +++ b/activitypub.py @@ -181,15 +181,17 @@ class MicroblogPubBackend(Backend): logger.info(f"{iri} found in cache") return ACTORS_CACHE[iri] - data = DB.actors.find_one({"remote_id": iri}) - if data: - logger.info(f"{iri} found in DB cache") - ACTORS_CACHE[iri] = data["data"] - return data["data"] + # data = DB.actors.find_one({"remote_id": iri}) + # if data: + # if ap._has_type(data["type"], ap.ACTOR_TYPES): + # logger.info(f"{iri} found in DB cache") + # ACTORS_CACHE[iri] = data["data"] + # return data["data"] data = self._fetch_iri(iri) logger.debug(f"_fetch_iri({iri!r}) == {data!r}") if ap._has_type(data["type"], ap.ACTOR_TYPES): + logger.debug(f"caching actor {iri}") # Cache the actor DB.actors.update_one( {"remote_id": iri}, From c01cc39e207e0ac55052d86aff9137bb0d50e176 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 17 Jul 2018 23:38:37 +0200 Subject: [PATCH 0228/1425] Template tweak --- app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app.py b/app.py index eebdbc6..494fcee 100644 --- a/app.py +++ b/app.py @@ -284,6 +284,8 @@ def get_actor(url): return get_backend().fetch_iri(url) except (ActivityNotFoundError, ActivityGoneError): return f"Deleted<{url}>" + except Exception: + return f"Error<{url}>" @app.template_filter() From ac5f349d0f290a7109e52940ffcb804e518c8a5b Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 17 Jul 2018 23:42:21 +0200 Subject: [PATCH 0229/1425] Add an config item to hide following --- app.py | 3 +++ config.py | 2 ++ templates/header.html | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 494fcee..44a0da1 100644 --- a/app.py +++ b/app.py @@ -1618,6 +1618,9 @@ def following(): ) ) + if config.HIDE_FOLLOWING: + abort(404) + following, older_than, newer_than = paginated_query(DB.activities, q) following = [ (doc["remote_id"], get_backend().fetch_iri(doc["activity"]["object"])) diff --git a/config.py b/config.py index 8c78fb6..9031b52 100644 --- a/config.py +++ b/config.py @@ -72,6 +72,8 @@ with open(os.path.join(KEY_DIR, "me.yml")) as f: PASS = conf["pass"] EXTRA_INBOXES = conf.get("extra_inboxes", []) + HIDE_FOLLOWING = strtobool(conf.get("hide_following", "false")) + # Theme-related config theme_conf = conf.get("theme", {}) THEME_STYLE = ThemeStyle(theme_conf.get("style", DEFAULT_THEME_STYLE)) diff --git a/templates/header.html b/templates/header.html index 1eb54a8..1d00568 100644 --- a/templates/header.html +++ b/templates/header.html @@ -16,7 +16,7 @@
  • With replies {{ with_replies_count }}
  • Liked {{ liked_count }}
  • Followers {{ followers_count }}
    • -
  • Following {{ following_count }}
    • +{% if not config.HIDE_FOLLOWING or session.logged_in %}
  • Following {{ following_count }}
    • {% endif %} From 13cee5dbaeb602b96e41454e71f8efda1a3f39d3 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 17 Jul 2018 23:46:12 +0200 Subject: [PATCH 0230/1425] Tweak the tasks flow --- app.py | 3 ++- config.py | 2 +- tasks.py | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 44a0da1..54d15c4 100644 --- a/app.py +++ b/app.py @@ -80,9 +80,10 @@ back = activitypub.MicroblogPubBackend() def save_cb(box: Box, iri: str) -> None: - tasks.cache_attachments.delay(iri) if box == Box.INBOX: tasks.process_new_activity.delay(iri) + else: + tasks.cache_attachments.delay(iri) back.set_save_cb(save_cb) diff --git a/config.py b/config.py index 9031b52..855f85b 100644 --- a/config.py +++ b/config.py @@ -72,7 +72,7 @@ with open(os.path.join(KEY_DIR, "me.yml")) as f: PASS = conf["pass"] EXTRA_INBOXES = conf.get("extra_inboxes", []) - HIDE_FOLLOWING = strtobool(conf.get("hide_following", "false")) + HIDE_FOLLOWING = conf.get("hide_following", False) # Theme-related config theme_conf = conf.get("theme", {}) diff --git a/tasks.py b/tasks.py index 79a1d51..603a797 100644 --- a/tasks.py +++ b/tasks.py @@ -54,6 +54,7 @@ def process_new_activity(self, iri: str) -> None: ) log.info(f"new activity {iri} processed") + cache_attachments.delay(iri) except (ActivityGoneError, ActivityNotFoundError): log.exception(f"dropping activity {iri}") except Exception as err: From 4877e955d03d339c695a280ed6f959e4bf3e900b Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 18 Jul 2018 00:04:40 +0200 Subject: [PATCH 0231/1425] More logging --- app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 54d15c4..c2bf01c 100644 --- a/app.py +++ b/app.py @@ -404,7 +404,7 @@ def is_api_request(): @app.errorhandler(ValueError) def handle_value_error(error): - logger.error(f"caught value error: {error!r}") + logger.error(f"caught value error: {error!r}, {error.__traceback__}") response = flask_jsonify(message=error.args[0]) response.status_code = 400 return response @@ -412,7 +412,7 @@ def handle_value_error(error): @app.errorhandler(Error) def handle_activitypub_error(error): - logger.error(f"caught activitypub error {error!r}") + logger.error(f"caught activitypub error {error!r}, {error.__traceback__}") response = flask_jsonify(error.to_dict()) response.status_code = error.status_code return response From 8b0a1ea368c48a14ad7d5ec01b3f98d513315ec1 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 18 Jul 2018 00:06:42 +0200 Subject: [PATCH 0232/1425] Make the "With replies" section private --- app.py | 1 + templates/header.html | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index c2bf01c..a4c84fa 100644 --- a/app.py +++ b/app.py @@ -728,6 +728,7 @@ def index(): @app.route("/with_replies") +@login_required def with_replies(): q = { "box": Box.OUTBOX.value, diff --git a/templates/header.html b/templates/header.html index 1d00568..388045c 100644 --- a/templates/header.html +++ b/templates/header.html @@ -13,7 +13,8 @@
    • Notes {{ notes_count }}
      • -
    • With replies {{ with_replies_count }}
      • +{% if session.logged_in %}
    • With replies {{ with_replies_count }}
      • +{% end %}
    • Liked {{ liked_count }}
    • Followers {{ followers_count }}
      • {% if not config.HIDE_FOLLOWING or session.logged_in %}
    • Following {{ following_count }}
      • {% endif %} From ecd598eed69c1820009520afd44aa63cbb7ab5f6 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 18 Jul 2018 00:11:05 +0200 Subject: [PATCH 0233/1425] CSS tweak --- sass/base_theme.scss | 6 ++++++ templates/header.html | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/sass/base_theme.scss b/sass/base_theme.scss index dae8140..df2c4b3 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -4,6 +4,12 @@ html, body { height: 100%; } + +@media only screen and (max-width: 480px) { + #menu-item-following { + display: none; + } +} body { background-color: $background-color; color: $color; diff --git a/templates/header.html b/templates/header.html index 388045c..eda8b41 100644 --- a/templates/header.html +++ b/templates/header.html @@ -14,10 +14,10 @@ From fcf7b9f939e54963b33d9b14ba386dcf7a618b76 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 18 Jul 2018 00:13:58 +0200 Subject: [PATCH 0234/1425] Fix the logging --- app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index a4c84fa..5317143 100644 --- a/app.py +++ b/app.py @@ -404,7 +404,7 @@ def is_api_request(): @app.errorhandler(ValueError) def handle_value_error(error): - logger.error(f"caught value error: {error!r}, {error.__traceback__}") + logger.error(f"caught value error: {error!r}, {error.__traceback__!r}") response = flask_jsonify(message=error.args[0]) response.status_code = 400 return response @@ -412,7 +412,7 @@ def handle_value_error(error): @app.errorhandler(Error) def handle_activitypub_error(error): - logger.error(f"caught activitypub error {error!r}, {error.__traceback__}") + logger.error(f"caught activitypub error {error!r}, {error.__traceback__!r}") response = flask_jsonify(error.to_dict()) response.status_code = error.status_code return response From ce4ff0b92ea95a76b651d2bbe95d2cedc5e4d815 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 18 Jul 2018 00:20:32 +0200 Subject: [PATCH 0235/1425] Fix exception logging --- app.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 5317143..41605dc 100644 --- a/app.py +++ b/app.py @@ -3,6 +3,7 @@ import json import logging import mimetypes import os +import traceback import urllib from datetime import datetime from datetime import timezone @@ -404,7 +405,9 @@ def is_api_request(): @app.errorhandler(ValueError) def handle_value_error(error): - logger.error(f"caught value error: {error!r}, {error.__traceback__!r}") + logger.error( + f"caught value error: {error!r}, {traceback.format_tb(error.__traceback__)}" + ) response = flask_jsonify(message=error.args[0]) response.status_code = 400 return response @@ -412,7 +415,9 @@ def handle_value_error(error): @app.errorhandler(Error) def handle_activitypub_error(error): - logger.error(f"caught activitypub error {error!r}, {error.__traceback__!r}") + logger.error( + f"caught activitypub error {error!r}, {traceback.format_tb(error.__traceback__)}" + ) response = flask_jsonify(error.to_dict()) response.status_code = error.status_code return response From 66c80e1084c6929956ebac7c82a60e33c527a73c Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 18 Jul 2018 00:29:45 +0200 Subject: [PATCH 0236/1425] Tweak the default config --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 855f85b..10b0bb5 100644 --- a/config.py +++ b/config.py @@ -72,7 +72,7 @@ with open(os.path.join(KEY_DIR, "me.yml")) as f: PASS = conf["pass"] EXTRA_INBOXES = conf.get("extra_inboxes", []) - HIDE_FOLLOWING = conf.get("hide_following", False) + HIDE_FOLLOWING = conf.get("hide_following", True) # Theme-related config theme_conf = conf.get("theme", {}) From 11fcf2dfb0ec386ab2c8b766b618efe651342715 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 18 Jul 2018 23:18:39 +0200 Subject: [PATCH 0237/1425] Fix the follower page --- app.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index 41605dc..a92d3b2 100644 --- a/app.py +++ b/app.py @@ -1601,8 +1601,13 @@ def followers(): ) ) - followers, older_than, newer_than = paginated_query(DB.activities, q) - followers = [get_backend().fetch_iri(doc["activity"]["actor"]) for doc in followers] + raw_followers, older_than, newer_than = paginated_query(DB.activities, q) + followers = [] + for doc in raw_followers: + try: + followers.append(get_backend().fetch_iri(doc["activity"]["actor"])) + except Exception: + pass return render_template( "followers.html", followers_data=followers, @@ -1625,7 +1630,7 @@ def following(): ) ) - if config.HIDE_FOLLOWING: + if config.HIDE_FOLLOWING and not session.get("logged_in", False): abort(404) following, older_than, newer_than = paginated_query(DB.activities, q) From 610262d7b6c057c1c25f677ec3714854fe5af2b9 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 20 Jul 2018 01:12:02 +0200 Subject: [PATCH 0238/1425] Properly cache the actor --- app.py | 18 ++++++++++++++---- tasks.py | 44 ++++++++++++++++++++++++++++++++++++++++---- templates/utils.html | 4 ++++ 3 files changed, 58 insertions(+), 8 deletions(-) diff --git a/app.py b/app.py index a92d3b2..50291d3 100644 --- a/app.py +++ b/app.py @@ -84,7 +84,7 @@ def save_cb(box: Box, iri: str) -> None: if box == Box.INBOX: tasks.process_new_activity.delay(iri) else: - tasks.cache_attachments.delay(iri) + tasks.cache_actor.delay(iri) back.set_save_cb(save_cb) @@ -674,6 +674,14 @@ def tmp_migrate4(): return "Done" +@app.route("/migration4") +@login_required +def tmp_migrate5(): + for activity in DB.activities.find(): + tasks.cache_actor.delay(activity["remote_id"], also_cache_attachments=False) + return "Done" + + def paginated_query(db, q, limit=25, sort_key="_id"): older_than = newer_than = None query_sort = -1 @@ -822,6 +830,7 @@ def note_by_id(note_id): DB.activities.find( { "meta.undo": False, + "meta.deleted": False, "type": ActivityType.LIKE.value, "$or": [ {"activity.object.id": data["activity"]["object"]["id"]}, @@ -830,12 +839,13 @@ def note_by_id(note_id): } ) ) - likes = [get_backend().fetch_iri(doc["activity"]["actor"]) for doc in likes] + likes = [doc["meta"]["actor"] for doc in likes] shares = list( DB.activities.find( { "meta.undo": False, + "meta.deleted": False, "type": ActivityType.ANNOUNCE.value, "$or": [ {"activity.object.id": data["activity"]["object"]["id"]}, @@ -844,7 +854,7 @@ def note_by_id(note_id): } ) ) - shares = [get_backend().fetch_iri(doc["activity"]["actor"]) for doc in shares] + shares = [doc["meta"]["actor"] for doc in shares] return render_template( "note.html", likes=likes, shares=shares, thread=thread, note=data @@ -1605,7 +1615,7 @@ def followers(): followers = [] for doc in raw_followers: try: - followers.append(get_backend().fetch_iri(doc["activity"]["actor"])) + followers.append(doc["meta"]["actor"]) except Exception: pass return render_template( diff --git a/tasks.py b/tasks.py index 603a797..3eccb3e 100644 --- a/tasks.py +++ b/tasks.py @@ -2,6 +2,8 @@ import json import logging import os import random +from typing import Dict +from typing import Any import requests from celery import Celery @@ -54,14 +56,48 @@ def process_new_activity(self, iri: str) -> None: ) log.info(f"new activity {iri} processed") - cache_attachments.delay(iri) + cache_actor.delay(iri) except (ActivityGoneError, ActivityNotFoundError): - log.exception(f"dropping activity {iri}") + log.exception(f"dropping activity {iri}, skip processing") except Exception as err: log.exception(f"failed to process new activity {iri}") self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) +def _actor_to_meta(actor: ap.BaseActivity) -> Dict[str, Any]: + return { + "url": actor.url, + "icon": actor.icon, + "name": actor.name, + "preferredUsername": actor.preferredUsername, + } + + +@app.task(bind=True, max_retries=12) +def cache_actor(self, iri: str, also_cache_attachments: bool = True) -> None: + try: + activity = ap.fetch_remote_activity(iri) + log.info(f"activity={activity!r}") + + actor = activity.get_actor() + + # Cache the actor info + DB.activities.update_one( + {"remote_id": iri}, {"$set": {"meta.actor": _actor_to_meta(actor)}} + ) + + log.info(f"actor cached for {iri}") + if also_cache_attachments: + cache_attachments.delay(iri) + + except (ActivityGoneError, ActivityNotFoundError): + DB.activities.update_one({"remote_id": iri}, {"$set": {"meta.deleted": True}}) + log.exception(f"flagging activity {iri} as deleted, no actor caching") + except Exception as err: + log.exception(f"failed to cache actor for {iri}") + self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) + + @app.task(bind=True, max_retries=12) def cache_attachments(self, iri: str) -> None: try: @@ -88,9 +124,9 @@ def cache_attachments(self, iri: str) -> None: log.info(f"attachments cached for {iri}") except (ActivityGoneError, ActivityNotFoundError): - log.exception(f"dropping activity {iri}") + log.exception(f"dropping activity {iri}, no attachment caching") except Exception as err: - log.exception(f"failed to process new activity {iri}") + log.exception(f"failed to cache attachments for {iri}") self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) diff --git a/templates/utils.html b/templates/utils.html index f78cada..3f267b5 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -16,7 +16,11 @@ {% macro display_note(obj, perma=False, ui=False, likes=[], shares=[], meta={}, no_color=False) -%} +{% if meta.actor %} +{% set actor = meta.actor %} +{% else %} {% set actor = obj.attributedTo | get_actor %} +{% endif %}
      From 3e7b09b7a854b0f2b21f3b81d43b3f6d1f362ef3 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 20 Jul 2018 10:56:39 +0200 Subject: [PATCH 0239/1425] More caching --- activitypub.py | 33 ++++++++++++++++++++++++++++++--- app.py | 25 +++++++++++++++++++++++++ tasks.py | 13 +------------ templates/index.html | 4 ++-- templates/liked.html | 2 +- templates/stream.html | 10 +++++----- 6 files changed, 64 insertions(+), 23 deletions(-) diff --git a/activitypub.py b/activitypub.py index 83e58ab..9a1d0b8 100644 --- a/activitypub.py +++ b/activitypub.py @@ -32,6 +32,15 @@ logger = logging.getLogger(__name__) ACTORS_CACHE = LRUCache(maxsize=256) +def _actor_to_meta(actor: ap.BaseActivity) -> Dict[str, Any]: + return { + "url": actor.url, + "icon": actor.icon, + "name": actor.name, + "preferredUsername": actor.preferredUsername, + } + + def _remove_id(doc: ap.ObjectType) -> ap.ObjectType: """Helper for removing MongoDB's `_id` field.""" doc = doc.copy() @@ -263,8 +272,15 @@ class MicroblogPubBackend(Backend): {"activity.object.id": obj.id}, {"$inc": {"meta.count_like": 1}, "$set": {"meta.liked": like.id}}, ) + DB.activities.update_one( - {"remote_id": like.id}, {"$set": {"meta.object": obj.to_dict(embed=True)}} + {"remote_id": like.id}, + { + "$set": { + "meta.object": obj.to_dict(embed=True), + "meta.object_actor": _actor_to_meta(obj.get_actor()), + } + }, ) @ensure_it_is_me @@ -291,7 +307,12 @@ class MicroblogPubBackend(Backend): obj = announce.get_object() DB.activities.update_one( {"remote_id": announce.id}, - {"$set": {"meta.object": obj.to_dict(embed=True)}}, + { + "$set": { + "meta.object": obj.to_dict(embed=True), + "meta.object_actor": _actor_to_meta(obj.get_actor()), + } + }, ) DB.activities.update_one( {"activity.object.id": obj.id}, {"$inc": {"meta.count_boost": 1}} @@ -313,8 +334,14 @@ class MicroblogPubBackend(Backend): obj = announce.get_object() DB.activities.update_one( {"remote_id": announce.id}, - {"$set": {"meta.object": obj.to_dict(embed=True)}}, + { + "$set": { + "meta.object": obj.to_dict(embed=True), + "meta.object_actor": _actor_to_meta(obj.get_actor()), + } + }, ) + DB.activities.update_one( {"activity.object.id": obj.id}, {"$set": {"meta.boosted": announce.id}} ) diff --git a/app.py b/app.py index 50291d3..00a290a 100644 --- a/app.py +++ b/app.py @@ -679,6 +679,31 @@ def tmp_migrate4(): def tmp_migrate5(): for activity in DB.activities.find(): tasks.cache_actor.delay(activity["remote_id"], also_cache_attachments=False) + + return "Done" + + +@app.route("/migration5") +@login_required +def tmp_migrate6(): + for activity in DB.activities.find(): + # tasks.cache_actor.delay(activity["remote_id"], also_cache_attachments=False) + try: + a = ap.parse_activity(activity["activity"]) + if a.has_type([ActivityType.LIKE, ActivityType.FOLLOW]): + DB.activities.update_one( + {"remote_id": a.id}, + { + "$set": { + "meta.object_actor": activitypub._actor_to_meta( + a.get_object().get_actor() + ) + } + }, + ) + except Exception: + app.logger.exception(f"processing {activity} failed") + return "Done" diff --git a/tasks.py b/tasks.py index 3eccb3e..f348ded 100644 --- a/tasks.py +++ b/tasks.py @@ -2,8 +2,6 @@ import json import logging import os import random -from typing import Dict -from typing import Any import requests from celery import Celery @@ -64,15 +62,6 @@ def process_new_activity(self, iri: str) -> None: self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) -def _actor_to_meta(actor: ap.BaseActivity) -> Dict[str, Any]: - return { - "url": actor.url, - "icon": actor.icon, - "name": actor.name, - "preferredUsername": actor.preferredUsername, - } - - @app.task(bind=True, max_retries=12) def cache_actor(self, iri: str, also_cache_attachments: bool = True) -> None: try: @@ -83,7 +72,7 @@ def cache_actor(self, iri: str, also_cache_attachments: bool = True) -> None: # Cache the actor info DB.activities.update_one( - {"remote_id": iri}, {"$set": {"meta.actor": _actor_to_meta(actor)}} + {"remote_id": iri}, {"$set": {"meta.actor": activitypub._actor_to_meta(actor)}} ) log.info(f"actor cached for {iri}") diff --git a/templates/index.html b/templates/index.html index a398087..55e64a5 100644 --- a/templates/index.html +++ b/templates/index.html @@ -24,7 +24,7 @@ {% for item in outbox_data %} {% if item | has_type('Announce') %} - {% set boost_actor = item.activity.actor | get_actor %} + {% set boost_actor = item.meta.actor %} {% if session.logged_in %}
      {{ boost_actor.name }} boosted @@ -41,7 +41,7 @@

      {% endif %} {% if item.meta.object %} - {{ utils.display_note(item.meta.object, ui=False) }} + {{ utils.display_note(item.meta.object, ui=False, meta={'actor': item.meta.object_actor}) }} {% endif %} {% elif item | has_type('Create') %} {{ utils.display_note(item.activity.object, meta=item.meta, no_color=True) }} diff --git a/templates/liked.html b/templates/liked.html index 29b903f..3548fd6 100644 --- a/templates/liked.html +++ b/templates/liked.html @@ -20,7 +20,7 @@ {% endif %} {% if item.meta.object %} - {{ utils.display_note(item.meta.object) }} + {{ utils.display_note(item.meta.object, meta={'actor': item.meta.object_actor}) }} {% endif %} {% endfor %} diff --git a/templates/stream.html b/templates/stream.html index 365288a..f12c0c7 100644 --- a/templates/stream.html +++ b/templates/stream.html @@ -1,6 +1,6 @@ {% extends "layout.html" %} {% import 'utils.html' as utils %} -{% block title %}Stream - {{ config.NAME }}{% endblock %} +{% block title %}{% if request.path == url_for('admin_stream') %}Stream{% else %}Notifications{% endif %} - {{ config.NAME }}{% endblock %} {% block content %}
      {% include "header.html" %} @@ -13,7 +13,7 @@ {% else %} {% if item | has_type('Announce') %} - {% set boost_actor = item.activity.actor | get_actor %} + {% set boost_actor = item.meta.actor %}

      {{ boost_actor.name or boost_actor.preferredUsername }} boosted

      {% if item.meta.object %} {{ utils.display_note(item.meta.object, ui=True) }} @@ -23,19 +23,19 @@ {% if item | has_type('Follow') %}

      new follower

      - {{ utils.display_actor_inline(item.activity.actor | get_actor, size=50) }} + {{ utils.display_actor_inline(item.meta.actor, size=50) }}
      {% elif item | has_type('Accept') %}

      you started following

      - {{ utils.display_actor_inline(item.activity.actor | get_actor, size=50) }} + {{ utils.display_actor_inline(item.meta.actor, size=50) }}
      {% elif item | has_type('Undo') %}

      unfollowed you

      - {{ utils.display_actor_inline(item.activity.actor | get_actor, size=50) }} + {{ utils.display_actor_inline(item.meta.actor, size=50) }}
      {% else %} From dc8365ff53ccdd34e97423eb1649b624379eb515 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 20 Jul 2018 12:14:11 +0200 Subject: [PATCH 0240/1425] Allow content negotiation for the note permalink --- app.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app.py b/app.py index 00a290a..9f1ea47 100644 --- a/app.py +++ b/app.py @@ -842,6 +842,9 @@ def _build_thread(data, include_children=True): @app.route("/note/") def note_by_id(note_id): + if is_api_request(): + return redirect(url_for('outbox_activity', item_id=note_id)) + data = DB.activities.find_one( {"box": Box.OUTBOX.value, "remote_id": back.activity_url(note_id)} ) From a3cb459ae05d83af6265016f4989eac408412e96 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 21 Jul 2018 00:04:15 +0200 Subject: [PATCH 0241/1425] Tweak the admin UI, start new lookup UI --- app.py | 14 +++++++++++++- templates/header.html | 11 ++++++++++- templates/layout.html | 2 ++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 9f1ea47..c830251 100644 --- a/app.py +++ b/app.py @@ -76,6 +76,7 @@ from config import _drop_db from config import custom_cache_purge_hook from utils.key import get_secret_key from utils.media import Kind +from utils.lookup import lookup back = activitypub.MicroblogPubBackend() @@ -843,7 +844,7 @@ def _build_thread(data, include_children=True): @app.route("/note/") def note_by_id(note_id): if is_api_request(): - return redirect(url_for('outbox_activity', item_id=note_id)) + return redirect(url_for("outbox_activity", item_id=note_id)) data = DB.activities.find_one( {"box": Box.OUTBOX.value, "remote_id": back.activity_url(note_id)} @@ -1229,6 +1230,17 @@ def admin(): ) +@app.route("/admin/lookup", methods=["GET", "POST"]) +@login_required +def admin_lookup(): + data = None + if request.method == "POST": + if request.form.get("url"): + data = lookup(request.form.get("url")) + + return render_template("lookup.html", data=data, url=request.form.get("url")) + + @app.route("/admin/thread") @login_required def admin_thread(): diff --git a/templates/header.html b/templates/header.html index eda8b41..287db10 100644 --- a/templates/header.html +++ b/templates/header.html @@ -1,15 +1,22 @@ diff --git a/templates/layout.html b/templates/layout.html index 42b2e95..a61a703 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -22,6 +22,8 @@
    • New
    • Stream
    • Notifications
      • +
    • Following
      • +
    • Lookup
    • Logout
    From ad355df6c599911414219c924a64b1763859f14d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 21 Jul 2018 00:15:47 +0200 Subject: [PATCH 0242/1425] Oops add missing code --- templates/lookup.html | 40 ++++++++++++++++++++++++++++++++++++++++ utils/lookup.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 templates/lookup.html create mode 100644 utils/lookup.py diff --git a/templates/lookup.html b/templates/lookup.html new file mode 100644 index 0000000..35af1b8 --- /dev/null +++ b/templates/lookup.html @@ -0,0 +1,40 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block title %}Lookup - {{ config.NAME }}{% endblock %} +{% block content %} +
    +{% include "header.html" %} +
    + + + + + + + +{% if data %} +{% set data = data.to_dict() %} +
    + {% if data | has_type('Person') or data | has_type('Service') %} +
    +
    + + + + +
    +
    + + + {{ utils.display_actor_inline(data, size=80) }} + {% elif data | has_type('Create') %} + {{ utils.display_note(data.object, ui=True) }} + {% elif data | has_type('Note') %} + {{ utils.display_note(data, ui=True) }} + {% endif %} +
    +{% endif %} + +
    +
    +{% endblock %} diff --git a/utils/lookup.py b/utils/lookup.py new file mode 100644 index 0000000..65c2c19 --- /dev/null +++ b/utils/lookup.py @@ -0,0 +1,32 @@ +import little_boxes.activitypub as ap +import json +import requests + +import mf2py + + +def lookup(url: str) -> ap.BaseActivity: + """Try to find an AP object related to the given URL.""" + backend = ap.get_backend() + resp = requests.get( + url, + timeout=15, + allow_redirects=False, + headers={"User-Agent": backend.user_agent()}, + ) + resp.raise_for_status() + + # If the page is HTML, maybe it contains an alternate link pointing to an AP object + for alternate in mf2py.parse(resp.text).get("alternates", []): + if alternate.get("type") == "application/activity+json": + return ap.fetch_remote_activity(alternate["url"]) + + try: + # Maybe the page was JSON-LD? + data = resp.json() + return ap.parse_activity(data) + except json.JSONDecodeError: + pass + + # Try content negotiation (retry with the AP Accept header) + return ap.fetch_remote_activity(url) From ea6be0ed8e5da5a500871d9b442c1f5c33ec4732 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 21 Jul 2018 01:05:51 +0200 Subject: [PATCH 0243/1425] Formatting --- .isort.cfg | 3 +++ app.py | 2 +- tasks.py | 3 ++- templates/utils.html | 1 + utils/lookup.py | 4 ++-- 5 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 .isort.cfg diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..0ed7634 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,3 @@ +[settings] +line_length=120 +force_single_line=true diff --git a/app.py b/app.py index c830251..91b6ab0 100644 --- a/app.py +++ b/app.py @@ -75,8 +75,8 @@ from config import VERSION from config import _drop_db from config import custom_cache_purge_hook from utils.key import get_secret_key -from utils.media import Kind from utils.lookup import lookup +from utils.media import Kind back = activitypub.MicroblogPubBackend() diff --git a/tasks.py b/tasks.py index f348ded..7824cd3 100644 --- a/tasks.py +++ b/tasks.py @@ -72,7 +72,8 @@ def cache_actor(self, iri: str, also_cache_attachments: bool = True) -> None: # Cache the actor info DB.activities.update_one( - {"remote_id": iri}, {"$set": {"meta.actor": activitypub._actor_to_meta(actor)}} + {"remote_id": iri}, + {"$set": {"meta.actor": activitypub._actor_to_meta(actor)}}, ) log.info(f"actor cached for {iri}") diff --git a/templates/utils.html b/templates/utils.html index 3f267b5..6d4b84a 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -172,6 +172,7 @@ {% macro display_thread(thread, likes=[], shares=[]) -%} {% for reply in thread %} +{{ reply }} {% if reply._requested %} {{ display_note(reply.activity.object, perma=True, ui=False, likes=likes, shares=shares, meta=reply.meta) }} {% else %} diff --git a/utils/lookup.py b/utils/lookup.py index 65c2c19..abd0d6f 100644 --- a/utils/lookup.py +++ b/utils/lookup.py @@ -1,8 +1,8 @@ -import little_boxes.activitypub as ap import json -import requests +import little_boxes.activitypub as ap import mf2py +import requests def lookup(url: str) -> ap.BaseActivity: From 8f7ee6f2cdf8a5c346ec95da63dd169250ecb0a5 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 21 Jul 2018 01:42:59 +0200 Subject: [PATCH 0244/1425] Forward replies --- tasks.py | 16 ++++++++++++++-- templates/utils.html | 1 - 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tasks.py b/tasks.py index 7824cd3..9656c84 100644 --- a/tasks.py +++ b/tasks.py @@ -15,6 +15,7 @@ from requests.exceptions import HTTPError import activitypub from config import DB from config import HEADERS +from config import ID from config import KEY from config import MEDIA_CACHE from config import USER_AGENT @@ -33,6 +34,7 @@ ap.use_backend(back) @app.task(bind=True, max_retries=12) def process_new_activity(self, iri: str) -> None: + """Process an activity received in the inbox""" try: activity = ap.fetch_remote_activity(iri) log.info(f"activity={activity!r}") @@ -45,7 +47,13 @@ def process_new_activity(self, iri: str) -> None: tag_stream = True elif activity.has_type(ap.ActivityType.CREATE): note = activity.get_object() - if not note.inReplyTo: + if note.inReplyTo: + reply = ap.fetch_remote_activity(note.inReplyTo) + if reply.id.startswith(ID) and activity.is_public(): + # The reply is public "local reply", forward the reply (i.e. the original activity) to the original + # recipients + activity.forward(reply.recipients()) + else: tag_stream = True log.info(f"{iri} tag_stream={tag_stream}") @@ -126,7 +134,11 @@ def post_to_inbox(self, payload: str, to: str) -> None: log.info("payload=%s", payload) log.info("generating sig") signed_payload = json.loads(payload) - generate_signature(signed_payload, KEY) + + # Don't overwrite the signature if we're forwarding an activity + if "signature" not in signed_payload: + generate_signature(signed_payload, KEY) + log.info("to=%s", to) resp = requests.post( to, diff --git a/templates/utils.html b/templates/utils.html index 6d4b84a..3f267b5 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -172,7 +172,6 @@ {% macro display_thread(thread, likes=[], shares=[]) -%} {% for reply in thread %} -{{ reply }} {% if reply._requested %} {{ display_note(reply.activity.object, perma=True, ui=False, likes=likes, shares=shares, meta=reply.meta) }} {% else %} From f05eb559f0a310425fe1365c0afbf2bcb47dcac3 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 21 Jul 2018 02:06:45 +0200 Subject: [PATCH 0245/1425] Partial ghost replies handling --- tasks.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/tasks.py b/tasks.py index 9656c84..f777820 100644 --- a/tasks.py +++ b/tasks.py @@ -32,7 +32,7 @@ back = activitypub.MicroblogPubBackend() ap.use_backend(back) -@app.task(bind=True, max_retries=12) +@app.task(bind=True, max_retries=12) # noqa: C901 def process_new_activity(self, iri: str) -> None: """Process an activity received in the inbox""" try: @@ -49,10 +49,28 @@ def process_new_activity(self, iri: str) -> None: note = activity.get_object() if note.inReplyTo: reply = ap.fetch_remote_activity(note.inReplyTo) - if reply.id.startswith(ID) and activity.is_public(): + if ( + reply.id.startswith(ID) or reply.has_mention(ID) + ) and activity.is_public(): # The reply is public "local reply", forward the reply (i.e. the original activity) to the original # recipients - activity.forward(reply.recipients()) + activity.forward(back.followers()) + + # (partial) Ghost replies handling + # [X] This is the first time the server has seen this Activity. + should_forward = False + local_followers = ID + "/followers" + for field in ["to", "cc"]: + if field in activity._data: + if local_followers in activity._data[field]: + # [X] The values of to, cc, and/or audience contain a Collection owned by the server. + should_forward = True + if not (note.inReplyTo and note.inReplyTo.startswith(ID)): + # [X] The values of inReplyTo, object, target and/or tag are objects owned by the server + should_forward = False + + if should_forward: + activity.forward(back.followers()) else: tag_stream = True From 4a22ac12a83369e1378427812cb8f8cccf40bebf Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 21 Jul 2018 11:41:49 +0200 Subject: [PATCH 0246/1425] Add webfinger support for the lookup, fix forwarding --- activitypub.py | 29 +++++++++++++++++++++++++++-- app.py | 1 + tasks.py | 30 +++++++++++++++++++++++++++--- utils/lookup.py | 6 ++++++ 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/activitypub.py b/activitypub.py index 9a1d0b8..ddd260a 100644 --- a/activitypub.py +++ b/activitypub.py @@ -32,13 +32,24 @@ logger = logging.getLogger(__name__) ACTORS_CACHE = LRUCache(maxsize=256) -def _actor_to_meta(actor: ap.BaseActivity) -> Dict[str, Any]: - return { +def _actor_to_meta(actor: ap.BaseActivity, with_inbox=False) -> Dict[str, Any]: + meta = { + "id": actor.id, "url": actor.url, "icon": actor.icon, "name": actor.name, "preferredUsername": actor.preferredUsername, } + if with_inbox: + meta.update( + { + "inbox": actor.inbox, + "sharedInbox": actor._data.get("endpoints", {}).get("sharedInbox"), + } + ) + logger.debug(f"meta={meta}") + + return meta def _remove_id(doc: ap.ObjectType) -> ap.ObjectType: @@ -120,6 +131,20 @@ class MicroblogPubBackend(Backend): } return [doc["activity"]["actor"] for doc in DB.activities.find(q)] + def followers_as_recipients(self) -> List[str]: + q = { + "box": Box.INBOX.value, + "type": ap.ActivityType.FOLLOW.value, + "meta.undo": False, + } + recipients = [] + for doc in DB.activities.find(q): + recipients.append( + doc["meta"]["actor"]["sharedInbox"] or doc["meta"]["actor"]["inbox"] + ) + + return list(set(recipients)) + def following(self) -> List[str]: q = { "box": Box.OUTBOX.value, diff --git a/app.py b/app.py index 91b6ab0..7b99cc1 100644 --- a/app.py +++ b/app.py @@ -689,6 +689,7 @@ def tmp_migrate5(): def tmp_migrate6(): for activity in DB.activities.find(): # tasks.cache_actor.delay(activity["remote_id"], also_cache_attachments=False) + try: a = ap.parse_activity(activity["activity"]) if a.has_type([ActivityType.LIKE, ActivityType.FOLLOW]): diff --git a/tasks.py b/tasks.py index f777820..2687545 100644 --- a/tasks.py +++ b/tasks.py @@ -54,7 +54,7 @@ def process_new_activity(self, iri: str) -> None: ) and activity.is_public(): # The reply is public "local reply", forward the reply (i.e. the original activity) to the original # recipients - activity.forward(back.followers()) + activity.forward(back.followers_as_recipients()) # (partial) Ghost replies handling # [X] This is the first time the server has seen this Activity. @@ -70,7 +70,7 @@ def process_new_activity(self, iri: str) -> None: should_forward = False if should_forward: - activity.forward(back.followers()) + activity.forward(back.followers_as_recipients()) else: tag_stream = True @@ -96,10 +96,34 @@ def cache_actor(self, iri: str, also_cache_attachments: bool = True) -> None: actor = activity.get_actor() + cache_actor_with_inbox = False + if activity.has_type(ap.ActivityType.FOLLOW): + if actor.id != ID: + # It's a Follow from the Inbox + cache_actor_with_inbox = True + else: + # It's a new following, cache the "object" (which is the actor we follow) + DB.activities.update_one( + {"remote_id": iri}, + { + "$set": { + "meta.object": activitypub._actor_to_meta( + activity.get_object() + ) + } + }, + ) + # Cache the actor info DB.activities.update_one( {"remote_id": iri}, - {"$set": {"meta.actor": activitypub._actor_to_meta(actor)}}, + { + "$set": { + "meta.actor": activitypub._actor_to_meta( + actor, cache_actor_with_inbox + ) + } + }, ) log.info(f"actor cached for {iri}") diff --git a/utils/lookup.py b/utils/lookup.py index abd0d6f..50267fb 100644 --- a/utils/lookup.py +++ b/utils/lookup.py @@ -3,10 +3,16 @@ import json import little_boxes.activitypub as ap import mf2py import requests +from little_boxes.webfinger import get_actor_url def lookup(url: str) -> ap.BaseActivity: """Try to find an AP object related to the given URL.""" + try: + return ap.fetch_remote_activity(get_actor_url(url)) + except Exception: + pass + backend = ap.get_backend() resp = requests.get( url, From 3ba14d938facf7ebe142bc5a684368ad52d4294a Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 21 Jul 2018 11:45:24 +0200 Subject: [PATCH 0247/1425] Use the cache for the following page --- app.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app.py b/app.py index 7b99cc1..b01992c 100644 --- a/app.py +++ b/app.py @@ -1685,10 +1685,7 @@ def following(): abort(404) following, older_than, newer_than = paginated_query(DB.activities, q) - following = [ - (doc["remote_id"], get_backend().fetch_iri(doc["activity"]["object"])) - for doc in following - ] + following = [doc["meta"]["object"] for doc in following] return render_template( "following.html", following_data=following, From 23faef985b96e36cf0294166210db4bf333a3d9d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 21 Jul 2018 21:05:35 +0200 Subject: [PATCH 0248/1425] Fix activity forwarding and the stream --- tasks.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/tasks.py b/tasks.py index 2687545..3416dce 100644 --- a/tasks.py +++ b/tasks.py @@ -41,12 +41,18 @@ def process_new_activity(self, iri: str) -> None: # Is the activity expected? # following = ap.get_backend().following() + should_forward = False tag_stream = False if activity.has_type(ap.ActivityType.ANNOUNCE): tag_stream = True + elif activity.has_type(ap.ActivityType.CREATE): note = activity.get_object() + # Make the note part of the stream if it's not a reply, or if it's a local reply + if not note.inReplyTo or note.inReplyTo.startswith(ID): + tag_stream = True + if note.inReplyTo: reply = ap.fetch_remote_activity(note.inReplyTo) if ( @@ -54,7 +60,7 @@ def process_new_activity(self, iri: str) -> None: ) and activity.is_public(): # The reply is public "local reply", forward the reply (i.e. the original activity) to the original # recipients - activity.forward(back.followers_as_recipients()) + should_forward = True # (partial) Ghost replies handling # [X] This is the first time the server has seen this Activity. @@ -65,18 +71,27 @@ def process_new_activity(self, iri: str) -> None: if local_followers in activity._data[field]: # [X] The values of to, cc, and/or audience contain a Collection owned by the server. should_forward = True + + # [X] The values of inReplyTo, object, target and/or tag are objects owned by the server if not (note.inReplyTo and note.inReplyTo.startswith(ID)): - # [X] The values of inReplyTo, object, target and/or tag are objects owned by the server should_forward = False - if should_forward: - activity.forward(back.followers_as_recipients()) - else: - tag_stream = True + elif activity.has_type(ap.ActivityType.DELETE): + note = DB.activities.find_one( + {"activity.object.id": activity.get_object().id} + ) + if note["meta"].get("forwarded", False): + # If the activity was originally forwarded, forward the delete too + should_forward = True + + if should_forward: + log.info(f"will forward {activity!r} to followers") + activity.forward(back.followers_as_recipients()) log.info(f"{iri} tag_stream={tag_stream}") DB.activities.update_one( - {"remote_id": activity.id}, {"$set": {"meta.stream": tag_stream}} + {"remote_id": activity.id}, + {"$set": {"meta.stream": tag_stream, "meta.forwarded": should_forward}}, ) log.info(f"new activity {iri} processed") From 648e385c49cd725999b9f80098e9d0ced42911de Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 21 Jul 2018 23:16:40 +0200 Subject: [PATCH 0249/1425] Add Open Graph metadata support --- app.py | 5 +++++ sass/base_theme.scss | 10 ++++++++-- tasks.py | 38 ++++++++++++++++++++++++++++++++++++++ templates/utils.html | 25 +++++++++++++++++++++++-- utils/media.py | 21 +++++++++++++++++++++ utils/opengraph.py | 19 +++---------------- 6 files changed, 98 insertions(+), 20 deletions(-) diff --git a/app.py b/app.py index b01992c..d868d95 100644 --- a/app.py +++ b/app.py @@ -238,6 +238,11 @@ def get_attachment_url(url, size): return _get_file_url(url, size, Kind.ATTACHMENT) +@app.template_filter() +def get_og_image_url(url, size=100): + return _get_file_url(url, size, Kind.OG_IMAGE) + + @app.template_filter() def permalink_id(val): return str(hash(val)) diff --git a/sass/base_theme.scss b/sass/base_theme.scss index df2c4b3..e7b5b71 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -189,9 +189,11 @@ a:hover { h3 { margin: 0; } } } +.note-box { + margin-bottom: 70px; +} .note { display: flex; - margin-bottom: 70px; .l { color: $color-note-link; } @@ -229,7 +231,11 @@ a:hover { padding:10px 0; } } - +.color-menu-background { + background: $color-menu-background; +} +.og-link { text-decoration: none; } +.og-link:hover { text-decoration: none; } .bar-item-no-hover { background: $color-menu-background; padding: 5px; diff --git a/tasks.py b/tasks.py index 3416dce..1ca0134 100644 --- a/tasks.py +++ b/tasks.py @@ -19,6 +19,7 @@ from config import ID from config import KEY from config import MEDIA_CACHE from config import USER_AGENT +from utils import opengraph from utils.media import Kind log = logging.getLogger(__name__) @@ -103,12 +104,49 @@ def process_new_activity(self, iri: str) -> None: self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) +@app.task(bind=True, max_retries=12) # noqa: C901 +def fetch_og_metadata(self, iri: str) -> None: + try: + activity = ap.fetch_remote_activity(iri) + log.info(f"activity={activity!r}") + if activity.has_type(ap.ActivityType.CREATE): + note = activity.get_object() + links = opengraph.links_from_note(note.to_dict()) + og_metadata = opengraph.fetch_og_metadata(USER_AGENT, links) + for og in og_metadata: + if not og.get("image"): + continue + MEDIA_CACHE.cache_og_image(og["image"]) + + log.debug(f"OG metadata {og_metadata!r}") + DB.activities.update_one( + {"remote_id": iri}, {"$set": {"meta.og_metadata": og_metadata}} + ) + + log.info(f"OG metadata fetched for {iri}") + except (ActivityGoneError, ActivityNotFoundError): + log.exception(f"dropping activity {iri}, skip OG metedata") + except requests.exceptions.HTTPError as http_err: + if 400 <= http_err.response.status_code < 500: + log.exception("bad request, no retry") + log.exception("failed to fetch OG metadata") + self.retry( + exc=http_err, countdown=int(random.uniform(2, 4) ** self.request.retries) + ) + except Exception as err: + log.exception(f"failed to fetch OG metadata for {iri}") + self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) + + @app.task(bind=True, max_retries=12) def cache_actor(self, iri: str, also_cache_attachments: bool = True) -> None: try: activity = ap.fetch_remote_activity(iri) log.info(f"activity={activity!r}") + if activity.has_type(ap.ActivityType.CREATE): + fetch_og_metadata.delay(iri) + actor = activity.get_actor() cache_actor_with_inbox = False diff --git a/templates/utils.html b/templates/utils.html index 3f267b5..db64c04 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -21,6 +21,7 @@ {% else %} {% set actor = obj.attributedTo | get_actor %} {% endif %} +
    @@ -63,6 +64,26 @@
    {% endif %} + + +{% if meta and meta.og_metadata %} +{% for og in meta.og_metadata %} + +
    + +
    +
    +{{ og.title }} +

    {{ og.description | truncate(80) }}

    +{{ og.site_name }} +
    +     +{% endfor %} +{% endif %} + + + +
    {% if perma %}{{ obj.published | format_time }} {% if not (obj.id | is_from_outbox) %} @@ -163,10 +184,10 @@ {% endif %} - +
    +
    -
    {%- endmacro %} diff --git a/utils/media.py b/utils/media.py index 13e2b0a..6767514 100644 --- a/utils/media.py +++ b/utils/media.py @@ -31,6 +31,7 @@ class Kind(Enum): ATTACHMENT = "attachment" ACTOR_ICON = "actor_icon" UPLOAD = "upload" + OG_IMAGE = "og" class MediaCache(object): @@ -38,6 +39,24 @@ class MediaCache(object): self.fs = gridfs.GridFS(gridfs_db) self.user_agent = user_agent + def cache_og_image(self, url: str) -> None: + if self.fs.find_one({"url": url, "kind": Kind.OG_IMAGE.value}): + return + i = load(url, self.user_agent) + # Save the original attachment (gzipped) + i.thumbnail((100, 100)) + with BytesIO() as buf: + with GzipFile(mode="wb", fileobj=buf) as f1: + i.save(f1, format=i.format) + buf.seek(0) + self.fs.put( + buf, + url=url, + size=100, + content_type=i.get_format_mimetype(), + kind=Kind.OG_IMAGE.value, + ) + def cache_attachment(self, url: str) -> None: if self.fs.find_one({"url": url, "kind": Kind.ATTACHMENT.value}): return @@ -141,6 +160,8 @@ class MediaCache(object): def cache(self, url: str, kind: Kind) -> None: if kind == Kind.ACTOR_ICON: self.cache_actor_icon(url) + elif kind == Kind.OG_IMAGE: + self.cache_og_image(url) else: self.cache_attachment(url) diff --git a/utils/opengraph.py b/utils/opengraph.py index b543269..762e5ef 100644 --- a/utils/opengraph.py +++ b/utils/opengraph.py @@ -23,24 +23,11 @@ def links_from_note(note): return links -def fetch_og_metadata(user_agent, col, remote_id): - doc = col.find_one({"remote_id": remote_id}) - if not doc: - raise ValueError - note = doc["activity"]["object"] - print(note) - links = links_from_note(note) - if not links: - return 0 - # FIXME(tsileo): set the user agent by giving HTML directly to OpenGraph +def fetch_og_metadata(user_agent, links): htmls = [] for l in links: check_url(l) - r = requests.get(l, headers={"User-Agent": user_agent}) + r = requests.get(l, headers={"User-Agent": user_agent}, timeout=15) r.raise_for_status() htmls.append(r.text) - links_og_metadata = [dict(opengraph.OpenGraph(html=html)) for html in htmls] - col.update_one( - {"remote_id": remote_id}, {"$set": {"meta.og_metadata": links_og_metadata}} - ) - return len(links) + return [dict(opengraph.OpenGraph(html=html)) for html in htmls] From a841efc9e84179abd4ca055fa624598d5384fb59 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 21 Jul 2018 23:24:12 +0200 Subject: [PATCH 0250/1425] Fix the following page --- templates/following.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/following.html b/templates/following.html index 1b96ed1..0acf561 100644 --- a/templates/following.html +++ b/templates/following.html @@ -8,12 +8,12 @@ {% include "header.html" %}
    - {% for (follow_id, followed) in following_data %} + {% for follow in following_data %} {% if session.logged_in %}
    - +
    @@ -21,7 +21,7 @@ {% endif %}
    - {{ utils.display_actor_inline(followed, size=80) }} + {{ utils.display_actor_inline(follow, size=80) }}
    {% endfor %} {{ utils.display_pagination(older_than, newer_than) }} From a165e363032bdfe7b6704330628dc4f7e7f9c0b8 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 22 Jul 2018 11:44:42 +0200 Subject: [PATCH 0251/1425] Tweak the lookup and the OG metedata tasks Now we don't fetch OG metadata for AP profile --- activitypub.py | 14 +++++++------- tasks.py | 1 + utils/lookup.py | 7 +++++-- utils/opengraph.py | 13 ++++++++++++- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/activitypub.py b/activitypub.py index ddd260a..02690ac 100644 --- a/activitypub.py +++ b/activitypub.py @@ -16,6 +16,7 @@ from little_boxes import strtobool from little_boxes.activitypub import _to_list from little_boxes.backend import Backend from little_boxes.errors import ActivityGoneError +from little_boxes.errors import NotAnActivityError from little_boxes.errors import Error from config import BASE_URL @@ -319,17 +320,16 @@ class MicroblogPubBackend(Backend): @ensure_it_is_me def inbox_announce(self, as_actor: ap.Person, announce: ap.Announce) -> None: - if isinstance(announce._data["object"], str) and not announce._data[ - "object" - ].startswith("http"): - # TODO(tsileo): actually drop it without storing it and better logging, also move the check somewhere else - # or remote it? - logger.warn( + # TODO(tsileo): actually drop it without storing it and better logging, also move the check somewhere else + # or remove it? + try: + obj = announce.get_object() + except NotAnActivityError: + logger.exception( f'received an Annouce referencing an OStatus notice ({announce._data["object"]}), dropping the message' ) return - obj = announce.get_object() DB.activities.update_one( {"remote_id": announce.id}, { diff --git a/tasks.py b/tasks.py index 1ca0134..8710422 100644 --- a/tasks.py +++ b/tasks.py @@ -6,6 +6,7 @@ import random import requests from celery import Celery from little_boxes import activitypub as ap +from little_boxes.errors import NotAnActivityError from little_boxes.errors import ActivityGoneError from little_boxes.errors import ActivityNotFoundError from little_boxes.httpsig import HTTPSigAuth diff --git a/utils/lookup.py b/utils/lookup.py index 50267fb..8e2d760 100644 --- a/utils/lookup.py +++ b/utils/lookup.py @@ -4,13 +4,16 @@ import little_boxes.activitypub as ap import mf2py import requests from little_boxes.webfinger import get_actor_url +from little_boxes.errors import NotAnActivityError def lookup(url: str) -> ap.BaseActivity: """Try to find an AP object related to the given URL.""" try: - return ap.fetch_remote_activity(get_actor_url(url)) - except Exception: + actor_url = get_actor_url(url) + if actor_url: + return ap.fetch_remote_activity(actor_url) + except NotAnActivityError: pass backend = ap.get_backend() diff --git a/utils/opengraph.py b/utils/opengraph.py index 762e5ef..b4bd704 100644 --- a/utils/opengraph.py +++ b/utils/opengraph.py @@ -1,8 +1,11 @@ import opengraph import requests from bs4 import BeautifulSoup +from little_boxes import activitypub as ap +from little_boxes.errors import NotAnActivityError from little_boxes.urlutils import check_url from little_boxes.urlutils import is_url_valid +from .lookup import lookup def links_from_note(note): @@ -10,7 +13,6 @@ def links_from_note(note): for t in note.get("tag", []): h = t.get("href") if h: - # TODO(tsileo): fetch the URL for Actor profile, type=mention tags_href.add(h) links = set() @@ -27,6 +29,15 @@ def fetch_og_metadata(user_agent, links): htmls = [] for l in links: check_url(l) + + # Remove any AP actor from the list + try: + p = lookup(l) + if p.has_type(ap.ACTOR_TYPES): + continue + except NotAnActivityError: + pass + r = requests.get(l, headers={"User-Agent": user_agent}, timeout=15) r.raise_for_status() htmls.append(r.text) From deea5be4521f0428a82f51d990bd5d67758dd80a Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 22 Jul 2018 12:04:18 +0200 Subject: [PATCH 0252/1425] Drop more OStatus stuff --- activitypub.py | 2 +- tasks.py | 40 ++++++++++++++++++++++++++++++---------- utils/lookup.py | 2 +- utils/opengraph.py | 1 + 4 files changed, 33 insertions(+), 12 deletions(-) diff --git a/activitypub.py b/activitypub.py index 02690ac..ed1a690 100644 --- a/activitypub.py +++ b/activitypub.py @@ -16,8 +16,8 @@ from little_boxes import strtobool from little_boxes.activitypub import _to_list from little_boxes.backend import Backend from little_boxes.errors import ActivityGoneError -from little_boxes.errors import NotAnActivityError from little_boxes.errors import Error +from little_boxes.errors import NotAnActivityError from config import BASE_URL from config import DB diff --git a/tasks.py b/tasks.py index 8710422..5793e8b 100644 --- a/tasks.py +++ b/tasks.py @@ -6,9 +6,9 @@ import random import requests from celery import Celery from little_boxes import activitypub as ap -from little_boxes.errors import NotAnActivityError from little_boxes.errors import ActivityGoneError from little_boxes.errors import ActivityNotFoundError +from little_boxes.errors import NotAnActivityError from little_boxes.httpsig import HTTPSigAuth from little_boxes.linked_data_sig import generate_signature from requests.exceptions import HTTPError @@ -44,10 +44,17 @@ def process_new_activity(self, iri: str) -> None: # Is the activity expected? # following = ap.get_backend().following() should_forward = False + should_delete = False tag_stream = False if activity.has_type(ap.ActivityType.ANNOUNCE): tag_stream = True + try: + activity.get_object() + except NotAnActivityError: + # Most likely on OStatus notice + tag_stream = False + should_delete = True elif activity.has_type(ap.ActivityType.CREATE): note = activity.get_object() @@ -56,13 +63,17 @@ def process_new_activity(self, iri: str) -> None: tag_stream = True if note.inReplyTo: - reply = ap.fetch_remote_activity(note.inReplyTo) - if ( - reply.id.startswith(ID) or reply.has_mention(ID) - ) and activity.is_public(): - # The reply is public "local reply", forward the reply (i.e. the original activity) to the original - # recipients - should_forward = True + try: + reply = ap.fetch_remote_activity(note.inReplyTo) + if ( + reply.id.startswith(ID) or reply.has_mention(ID) + ) and activity.is_public(): + # The reply is public "local reply", forward the reply (i.e. the original activity) to the + # original recipients + should_forward = True + except NotAnActivityError: + # Most likely a reply to an OStatus notce + should_delete = True # (partial) Ghost replies handling # [X] This is the first time the server has seen this Activity. @@ -82,7 +93,7 @@ def process_new_activity(self, iri: str) -> None: note = DB.activities.find_one( {"activity.object.id": activity.get_object().id} ) - if note["meta"].get("forwarded", False): + if note and note["meta"].get("forwarded", False): # If the activity was originally forwarded, forward the delete too should_forward = True @@ -90,10 +101,19 @@ def process_new_activity(self, iri: str) -> None: log.info(f"will forward {activity!r} to followers") activity.forward(back.followers_as_recipients()) + if should_delete: + log.info(f"will soft delete {activity!r}") + log.info(f"{iri} tag_stream={tag_stream}") DB.activities.update_one( {"remote_id": activity.id}, - {"$set": {"meta.stream": tag_stream, "meta.forwarded": should_forward}}, + { + "$set": { + "meta.stream": tag_stream, + "meta.forwarded": should_forward, + "meta.deleted": should_delete, + } + }, ) log.info(f"new activity {iri} processed") diff --git a/utils/lookup.py b/utils/lookup.py index 8e2d760..3bb4756 100644 --- a/utils/lookup.py +++ b/utils/lookup.py @@ -3,8 +3,8 @@ import json import little_boxes.activitypub as ap import mf2py import requests -from little_boxes.webfinger import get_actor_url from little_boxes.errors import NotAnActivityError +from little_boxes.webfinger import get_actor_url def lookup(url: str) -> ap.BaseActivity: diff --git a/utils/opengraph.py b/utils/opengraph.py index b4bd704..87ffa5e 100644 --- a/utils/opengraph.py +++ b/utils/opengraph.py @@ -5,6 +5,7 @@ from little_boxes import activitypub as ap from little_boxes.errors import NotAnActivityError from little_boxes.urlutils import check_url from little_boxes.urlutils import is_url_valid + from .lookup import lookup From dbabbfc55d09512fcc443315dfe84cb855f781bf Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 22 Jul 2018 12:17:55 +0200 Subject: [PATCH 0253/1425] Fix login flow --- app.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index d868d95..91258c5 100644 --- a/app.py +++ b/app.py @@ -23,6 +23,7 @@ import timeago from bson.objectid import ObjectId from dateutil import parser from flask import Flask +from flask import make_response from flask import Response from flask import abort from flask import jsonify as flask_jsonify @@ -345,11 +346,30 @@ def is_img(filename): return _is_img(filename) +def add_response_headers(headers={}): + """This decorator adds the headers passed in to the response""" + def decorator(f): + @wraps(f) + def decorated_function(*args, **kwargs): + resp = make_response(f(*args, **kwargs)) + h = resp.headers + for header, value in headers.items(): + h[header] = value + return resp + return decorated_function + return decorator + + +def noindex(f): + """This decorator passes X-Robots-Tag: noindex, nofollow""" + return add_response_headers({'X-Robots-Tag': 'noindex, nofollow'})(f) + + def login_required(f): @wraps(f) def decorated_function(*args, **kwargs): if not session.get("logged_in"): - return redirect(url_for("login", next=request.url)) + return redirect(url_for("admin_login", next=request.url)) return f(*args, **kwargs) return decorated_function @@ -445,6 +465,7 @@ def robots_txt(): @app.route("/media/") +@noindex def serve_media(media_id): f = MEDIA_CACHE.fs.get(ObjectId(media_id)) resp = app.response_class(f, direct_passthrough=True, mimetype=f.content_type) @@ -484,6 +505,7 @@ def admin_logout(): @app.route("/login", methods=["POST", "GET"]) +@noindex def admin_login(): if session.get("logged_in") is True: return redirect(url_for("admin_notifications")) @@ -1840,7 +1862,7 @@ def indieauth_flow(): def indieauth_endpoint(): if request.method == "GET": if not session.get("logged_in"): - return redirect(url_for("login", next=request.url)) + return redirect(url_for("admin_login", next=request.url)) me = request.args.get("me") # FIXME(tsileo): ensure me == ID From f92a6ea3dc84d0655a236262701bfdd5ff4f75ca Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 22 Jul 2018 12:25:56 +0200 Subject: [PATCH 0254/1425] Improve error logging --- app.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app.py b/app.py index 91258c5..61c65d5 100644 --- a/app.py +++ b/app.py @@ -449,6 +449,16 @@ def handle_activitypub_error(error): return response +@app.errorhandler(Exception) +def handle_other_error(error): + logger.error( + f"caught error {error!r}, {traceback.format_tb(error.__traceback__)}" + ) + response = flask_jsonify({}) + response.status_code = 500 + return response + + # App routes ROBOTS_TXT = """User-agent: * From 42466ddfba4d47c9ceed462b7cd33ba5b5b2fc0d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 22 Jul 2018 12:42:36 +0200 Subject: [PATCH 0255/1425] Support actor without URL --- app.py | 23 +++++++++++++++-------- templates/utils.html | 10 +++++----- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/app.py b/app.py index 61c65d5..5791b18 100644 --- a/app.py +++ b/app.py @@ -274,6 +274,13 @@ def domain(url): return urlparse(url).netloc +@app.template_filter() +def url_or_id(d): + if 'url' in d: + return d['url'] + return d['id'] + + @app.template_filter() def get_url(u): if isinstance(u, dict): @@ -449,14 +456,14 @@ def handle_activitypub_error(error): return response -@app.errorhandler(Exception) -def handle_other_error(error): - logger.error( - f"caught error {error!r}, {traceback.format_tb(error.__traceback__)}" - ) - response = flask_jsonify({}) - response.status_code = 500 - return response +# @app.errorhandler(Exception) +# def handle_other_error(error): +# logger.error( +# f"caught error {error!r}, {traceback.format_tb(error.__traceback__)}" +# ) +# response = flask_jsonify({}) +# response.status_code = 500 +# return response # App routes diff --git a/templates/utils.html b/templates/utils.html index db64c04..c3b586e 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -1,5 +1,5 @@ {% macro display_actor_inline(follower, size=50) -%} - + {% if not follower.icon %} @@ -8,7 +8,7 @@
    {{ follower.name or follower.preferredUsername }}
    -@{{ follower.preferredUsername }}@{{ follower.url | get_url | domain }} +@{{ follower.preferredUsername }}@{{ follower | url_or_id | get_url | domain }}
    {%- endmacro %} @@ -25,14 +25,14 @@
    - +
    - {{ actor.name or actor.preferredUsername }} - @{% if not no_color and obj.id | is_from_outbox %}{{ actor.preferredUsername }}{% else %}{{ actor.preferredUsername }}{% endif %}@{% if not no_color and obj.id | is_from_outbox %}{{ actor.url | get_url | domain }}{% else %}{{ actor.url | get_url | domain }}{% endif %} + {{ actor.name or actor.preferredUsername }} + @{% if not no_color and obj.id | is_from_outbox %}{{ actor.preferredUsername }}{% else %}{{ actor.preferredUsername }}{% endif %}@{% if not no_color and obj.id | is_from_outbox %}{{ actor | url_or_id | get_url | domain }}{% else %}{{ actor | url_or_id | get_url | domain }}{% endif %} {% if not perma %} From fcf9524ac090f4594177457ca16dbcfe8e8f4daf Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 22 Jul 2018 12:53:09 +0200 Subject: [PATCH 0256/1425] Tweaks --- tasks.py | 3 ++- templates/utils.html | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tasks.py b/tasks.py index 5793e8b..d7f5f05 100644 --- a/tasks.py +++ b/tasks.py @@ -117,7 +117,8 @@ def process_new_activity(self, iri: str) -> None: ) log.info(f"new activity {iri} processed") - cache_actor.delay(iri) + if not should_delete: + cache_actor.delay(iri) except (ActivityGoneError, ActivityNotFoundError): log.exception(f"dropping activity {iri}, skip processing") except Exception as err: diff --git a/templates/utils.html b/templates/utils.html index c3b586e..c428fe6 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -1,4 +1,5 @@ {% macro display_actor_inline(follower, size=50) -%} +{% if follower and follower.id %} {% if not follower.icon %} @@ -11,6 +12,7 @@ @{{ follower.preferredUsername }}@{{ follower | url_or_id | get_url | domain }}
    +{% endif %} {%- endmacro %} From e2d814a456cba71c873b53c379fcc5037e9c467a Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 22 Jul 2018 21:34:42 +0200 Subject: [PATCH 0257/1425] Use the toot namespace/AP extension --- app.py | 18 ++++++++---------- config.py | 7 ++----- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/app.py b/app.py index 5791b18..b927d47 100644 --- a/app.py +++ b/app.py @@ -276,9 +276,9 @@ def domain(url): @app.template_filter() def url_or_id(d): - if 'url' in d: - return d['url'] - return d['id'] + if "url" in d: + return d["url"] + return d["id"] @app.template_filter() @@ -355,6 +355,7 @@ def is_img(filename): def add_response_headers(headers={}): """This decorator adds the headers passed in to the response""" + def decorator(f): @wraps(f) def decorated_function(*args, **kwargs): @@ -363,13 +364,15 @@ def add_response_headers(headers={}): for header, value in headers.items(): h[header] = value return resp + return decorated_function + return decorator def noindex(f): """This decorator passes X-Robots-Tag: noindex, nofollow""" - return add_response_headers({'X-Robots-Tag': 'noindex, nofollow'})(f) + return add_response_headers({"X-Robots-Tag": "noindex, nofollow"})(f) def login_required(f): @@ -415,7 +418,7 @@ def api_required(f): def jsonify(**data): if "@context" not in data: - data["@context"] = config.CTX_AS + data["@context"] = config.DEFAULT_CTX return Response( response=json.dumps(data), headers={ @@ -978,11 +981,6 @@ def wellknown_nodeinfo(): ) -# @app.route('/fake_feed') -# def fake_feed(): -# return 'https://lol3.tun.a4.io/fake_feedhttps://lol3.tun.a4.io/fake' - - @app.route("/.well-known/webfinger") def wellknown_webfinger(): """Enable WebFinger support, required for Mastodon interopability.""" diff --git a/config.py b/config.py index 10b0bb5..d824611 100644 --- a/config.py +++ b/config.py @@ -9,6 +9,7 @@ import sass import yaml from itsdangerous import JSONWebSignatureSerializer from little_boxes import strtobool +from little_boxes.activitypub import DEFAULT_CTX from pymongo import MongoClient from utils.key import KEY_DIR @@ -46,10 +47,6 @@ VERSION = ( DEBUG_MODE = strtobool(os.getenv("MICROBLOGPUB_DEBUG", "false")) - -CTX_AS = "https://www.w3.org/ns/activitystreams" -CTX_SECURITY = "https://w3id.org/security/v1" -AS_PUBLIC = "https://www.w3.org/ns/activitystreams#Public" HEADERS = [ "application/activity+json", "application/ld+json;profile=https://www.w3.org/ns/activitystreams", @@ -129,7 +126,7 @@ def _admin_jwt_token() -> str: ADMIN_API_KEY = get_secret_key("admin_api_key", _admin_jwt_token) ME = { - "@context": [CTX_AS, CTX_SECURITY], + "@context": DEFAULT_CTX, "type": "Person", "id": ID, "following": ID + "/following", From d26d6ba70e8dceec111ec81c7aff93838e9da398 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 22 Jul 2018 21:35:42 +0200 Subject: [PATCH 0258/1425] Formatting --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index b927d47..b2dd57c 100644 --- a/app.py +++ b/app.py @@ -23,10 +23,10 @@ import timeago from bson.objectid import ObjectId from dateutil import parser from flask import Flask -from flask import make_response from flask import Response from flask import abort from flask import jsonify as flask_jsonify +from flask import make_response from flask import redirect from flask import render_template from flask import request From f23dbcaf05ee451ca255953acb0c44b45588dc97 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 22 Jul 2018 21:54:24 +0200 Subject: [PATCH 0259/1425] Add support for the featured collection --- activitypub.py | 11 +++++++++++ app.py | 9 ++++++++- config.py | 1 + 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/activitypub.py b/activitypub.py index ed1a690..ec621e9 100644 --- a/activitypub.py +++ b/activitypub.py @@ -638,6 +638,16 @@ def embed_collection(total_items, first_page_id): } +def simple_build_ordered_collection(col_name, data): + return { + "@context": ap.COLLECTION_CTX, + "id": BASE_URL + "/" + col_name, + "totalItems": len(data), + "type": ap.ActivityType.ORDERED_COLLECTION.value, + "orederedItems": data, + } + + def build_ordered_collection( col, q=None, cursor=None, map_func=None, limit=50, col_name=None, first_page=False ): @@ -652,6 +662,7 @@ def build_ordered_collection( if not data: return { + "@context": ap.COLLECTION_CTX, "id": BASE_URL + "/" + col_name, "totalItems": 0, "type": ap.ActivityType.ORDERED_COLLECTION.value, diff --git a/app.py b/app.py index b2dd57c..07bcdaa 100644 --- a/app.py +++ b/app.py @@ -23,10 +23,10 @@ import timeago from bson.objectid import ObjectId from dateutil import parser from flask import Flask +from flask import make_response from flask import Response from flask import abort from flask import jsonify as flask_jsonify -from flask import make_response from flask import redirect from flask import render_template from flask import request @@ -1779,6 +1779,13 @@ def tags(tag): ) +@app.route("/featured") +def featured(): + if not is_api_request(): + abort(404) + return jsonify(**activitypub.simple_build_ordered_collection("featured", [])) + + @app.route("/liked") def liked(): if not is_api_request(): diff --git a/config.py b/config.py index d824611..db39fb0 100644 --- a/config.py +++ b/config.py @@ -131,6 +131,7 @@ ME = { "id": ID, "following": ID + "/following", "followers": ID + "/followers", + "featured": ID + "/featured", "liked": ID + "/liked", "inbox": ID + "/inbox", "outbox": ID + "/outbox", From a5e0983ca8c6f1810b4d0da5d984d73c85682427 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 22 Jul 2018 22:22:30 +0200 Subject: [PATCH 0260/1425] Finish support for pinned/featured note --- app.py | 51 +++++++++++++++++++++++++++++++++++++++++--- sass/base_theme.scss | 16 +++++++++++++- templates/index.html | 10 +++++++++ templates/utils.html | 20 +++++++++++++++-- 4 files changed, 91 insertions(+), 6 deletions(-) diff --git a/app.py b/app.py index 07bcdaa..7f226d5 100644 --- a/app.py +++ b/app.py @@ -803,14 +803,25 @@ def index(): "activity.object.inReplyTo": None, "meta.deleted": False, "meta.undo": False, + "$or": [{"meta.pinned": False}, {"meta.pinned": {"$exists": False}}], } - outbox_data, older_than, newer_than = paginated_query(DB.activities, q) - + q_pinned = { + "box": Box.OUTBOX.value, + "type": ActivityType.CREATE.value, + "meta.deleted": False, + "meta.undo": False, + "meta.pinned": True, + } + pinned = list(DB.activities.find(q_pinned)) + outbox_data, older_than, newer_than = paginated_query( + DB.activities, q, limit=25 - len(pinned) + ) return render_template( "index.html", outbox_data=outbox_data, older_than=older_than, newer_than=newer_than, + pinned=pinned, ) @@ -1453,6 +1464,32 @@ def api_like(): return _user_api_response(activity=like.id) +@app.route("/api/note/pin", methods=["POST"]) +@api_required +def api_pin(): + note = _user_api_get_note(from_outbox=True) + + DB.activities.update_one( + {"activity.object.id": note.id, "box": Box.OUTBOX.value}, + {"$set": {"meta.pinned": True}}, + ) + + return _user_api_response(pinned=True) + + +@app.route("/api/note/unpin", methods=["POST"]) +@api_required +def api_unpin(): + note = _user_api_get_note(from_outbox=True) + + DB.activities.update_one( + {"activity.object.id": note.id, "box": Box.OUTBOX.value}, + {"$set": {"meta.pinned": False}}, + ) + + return _user_api_response(pinned=False) + + @app.route("/api/undo", methods=["POST"]) @api_required def api_undo(): @@ -1783,7 +1820,15 @@ def tags(tag): def featured(): if not is_api_request(): abort(404) - return jsonify(**activitypub.simple_build_ordered_collection("featured", [])) + q = { + "box": Box.OUTBOX.value, + "type": ActivityType.CREATE.value, + "meta.deleted": False, + "meta.undo": False, + "meta.pinned": True, + } + data = [clean_activity(doc["activity"]["object"]) for doc in DB.activities.find(q)] + return jsonify(**activitypub.simple_build_ordered_collection("featured", data)) @app.route("/liked") diff --git a/sass/base_theme.scss b/sass/base_theme.scss index e7b5b71..85447cd 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -237,13 +237,27 @@ a:hover { .og-link { text-decoration: none; } .og-link:hover { text-decoration: none; } .bar-item-no-hover { + cursor: default; background: $color-menu-background; padding: 5px; color: $color-light; margin-right:5px; border-radius:2px; } - +.bar-item-no-hover:hover { + cursor: default; +} +.bar-item-pinned { + cursor: default; + background: $color-menu-background; + color: $color-light; + padding: 5px; + margin-right:5px; + border-radius:2px; +} +.bar-item-pinned:hover { + cursor: default; +} .bar-item { background: $color-menu-background; padding: 5px; diff --git a/templates/index.html b/templates/index.html index 55e64a5..1331a1f 100644 --- a/templates/index.html +++ b/templates/index.html @@ -21,6 +21,16 @@ {% include "header.html" %}
    + {% for item in pinned %} + {% if item.meta.pinned %} +

    + pinned +

    + {% endif %} + + {{ utils.display_note(item.activity.object, meta=item.meta, no_color=True) }} + {% endfor %} + {% for item in outbox_data %} {% if item | has_type('Announce') %} diff --git a/templates/utils.html b/templates/utils.html index c428fe6..ebf5f23 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -105,8 +105,8 @@ {% elif meta.count_reply and session.logged_in %} {{ meta.count_reply }} replies{% endif %} -{% if meta.count_boost and obj.id | is_from_outbox %}{{ meta.count_boost }} boosts{% endif %} -{% if meta.count_like and obj.id | is_from_outbox %}{{ meta.count_like }} likes{% endif %} +{% if not perma and meta.count_boost and obj.id | is_from_outbox %}{{ meta.count_boost }} boosts{% endif %} +{% if not perma and meta.count_like and obj.id | is_from_outbox %}{{ meta.count_like }} likes{% endif %} {% if session.logged_in %} {% if ui%} @@ -153,6 +153,22 @@ +{% if meta.pinned %} +
    + + + + +
    +{% else %} +
    + + + + +
    +{% endif %} + {% else %}
    From f7a5f56955ce420bc2424995cd865c49d140d1b4 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 22 Jul 2018 22:25:28 +0200 Subject: [PATCH 0261/1425] Fix the index --- app.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/app.py b/app.py index 7f226d5..9d6d127 100644 --- a/app.py +++ b/app.py @@ -805,17 +805,23 @@ def index(): "meta.undo": False, "$or": [{"meta.pinned": False}, {"meta.pinned": {"$exists": False}}], } - q_pinned = { - "box": Box.OUTBOX.value, - "type": ActivityType.CREATE.value, - "meta.deleted": False, - "meta.undo": False, - "meta.pinned": True, - } - pinned = list(DB.activities.find(q_pinned)) + + pinned = [] + # Only fetch the pinned notes if we're on the first page + if not request.args.get("older_than") and not request.args.get("newer_than"): + q_pinned = { + "box": Box.OUTBOX.value, + "type": ActivityType.CREATE.value, + "meta.deleted": False, + "meta.undo": False, + "meta.pinned": True, + } + pinned = list(DB.activities.find(q_pinned)) + outbox_data, older_than, newer_than = paginated_query( DB.activities, q, limit=25 - len(pinned) ) + return render_template( "index.html", outbox_data=outbox_data, From 7cf6027bf9d0e4fabcec67d062385dc7d4f0c929 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 22 Jul 2018 22:28:40 +0200 Subject: [PATCH 0262/1425] Document the new pin/unpin API endpoint --- README.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/README.md b/README.md index 338016a..32a26fa 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,50 @@ $ http POST https://microblog.pub/api/note/delete Authorization:'Bearer ' } ``` +### POST /api/note/pin{?id} + +Adds the given note `id` (the note must from the instance outbox) to the featured collection (and pins it on the homepage). + +Answers a **201** (Created) status code. + +You can pass the `id` via JSON, form data or query argument. + +#### Example + +```shell +$ http POST https://microblog.pub/api/note/pin Authorization:'Bearer ' id=http://microblob.pub/outbox//activity +``` + +#### Response + +```json +{ + "pinned": true +} +``` + +### POST /api/note/unpin{?id} + +Removes the given note `id` (the note must from the instance outbox) from the featured collection (and un-pins it). + +Answers a **201** (Created) status code. + +You can pass the `id` via JSON, form data or query argument. + +#### Example + +```shell +$ http POST https://microblog.pub/api/note/unpin Authorization:'Bearer ' id=http://microblob.pub/outbox//activity +``` + +#### Response + +```json +{ + "pinned": false +} +``` + ### POST /api/like{?id} Likes the given activity. From b4e5bda62e283fe61abe29328c5eafb732f1bb7a Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 23 Jul 2018 08:30:51 +0200 Subject: [PATCH 0263/1425] Tweak/debug --- activitypub.py | 13 ++----------- app.py | 5 ++++- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/activitypub.py b/activitypub.py index ec621e9..43d0576 100644 --- a/activitypub.py +++ b/activitypub.py @@ -384,26 +384,17 @@ class MicroblogPubBackend(Backend): @ensure_it_is_me def inbox_delete(self, as_actor: ap.Person, delete: ap.Delete) -> None: obj = delete.get_object() + logger.debug("delete object={obj!r}") DB.activities.update_one( {"activity.object.id": obj.id}, {"$set": {"meta.deleted": True}} ) - if obj.ACTIVITY_TYPE != ap.ActivityType.NOTE: - obj = ap.parse_activity( - DB.activities.find_one( - { - "activity.object.id": delete.get_object().id, - "type": ap.ActivityType.CREATE.value, - } - )["activity"] - ).get_object() - logger.info(f"inbox_delete handle_replies obj={obj!r}") # Fake a Undo so any related Like/Announce doesn't appear on the web UI DB.activities.update( {"meta.object.id": obj.id}, - {"$set": {"meta.undo": True, "meta.exta": "object deleted"}}, + {"$set": {"meta.undo": True, "meta.extra": "object deleted"}}, ) if obj: self._handle_replies_delete(as_actor, obj) diff --git a/app.py b/app.py index 9d6d127..a850d7f 100644 --- a/app.py +++ b/app.py @@ -241,7 +241,10 @@ def get_attachment_url(url, size): @app.template_filter() def get_og_image_url(url, size=100): - return _get_file_url(url, size, Kind.OG_IMAGE) + try: + return _get_file_url(url, size, Kind.OG_IMAGE) + except Exception: + return '' @app.template_filter() From 0cc5122a87d704471f61d9289d3437ed6233eb65 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 23 Jul 2018 21:43:03 +0200 Subject: [PATCH 0264/1425] Fix typo --- activitypub.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/activitypub.py b/activitypub.py index 43d0576..4dcb309 100644 --- a/activitypub.py +++ b/activitypub.py @@ -635,7 +635,7 @@ def simple_build_ordered_collection(col_name, data): "id": BASE_URL + "/" + col_name, "totalItems": len(data), "type": ap.ActivityType.ORDERED_COLLECTION.value, - "orederedItems": data, + "orderedItems": data, } @@ -657,7 +657,7 @@ def build_ordered_collection( "id": BASE_URL + "/" + col_name, "totalItems": 0, "type": ap.ActivityType.ORDERED_COLLECTION.value, - "orederedItems": [], + "oredredItems": [], } start_cursor = str(data[0]["_id"]) From c55cfa1f8ad2a3eae8d2323dde66aca6373ed1f8 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 23 Jul 2018 22:11:03 +0200 Subject: [PATCH 0265/1425] Add stream debug option --- app.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/app.py b/app.py index a850d7f..2b37046 100644 --- a/app.py +++ b/app.py @@ -229,6 +229,13 @@ def _get_file_url(url, size, kind): return _get_file_url(url, size, kind) +@app.template_filter() +def remove_mongo_id(dat): + if "_id" in dat: + del dat["_id"] + return dat + + @app.template_filter() def get_actor_icon_url(url, size): return _get_file_url(url, size, Kind.ACTOR_ICON) @@ -244,7 +251,7 @@ def get_og_image_url(url, size=100): try: return _get_file_url(url, size, Kind.OG_IMAGE) except Exception: - return '' + return "" @app.template_filter() @@ -1526,11 +1533,12 @@ def admin_stream(): q = {"meta.stream": True, "meta.deleted": False} inbox_data, older_than, newer_than = paginated_query(DB.activities, q) + tpl = "stream.html" + if request.args.get("debug"): + tpl = "stream_debug.html" + return render_template( - "stream.html", - inbox_data=inbox_data, - older_than=older_than, - newer_than=newer_than, + tpl, inbox_data=inbox_data, older_than=older_than, newer_than=newer_than ) From 2f25540092ee00fe9cd27dd702f93b6feab870ea Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 23 Jul 2018 22:15:07 +0200 Subject: [PATCH 0266/1425] Oops add missing template --- templates/stream_debug.html | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 templates/stream_debug.html diff --git a/templates/stream_debug.html b/templates/stream_debug.html new file mode 100644 index 0000000..6b4a96e --- /dev/null +++ b/templates/stream_debug.html @@ -0,0 +1,19 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block title %}{% if request.path == url_for('admin_stream') %}Stream{% else %}Notifications{% endif %} - {{ config.NAME }}{% endblock %} +{% block content %} +
    +{% include "header.html" %} +
    + +
    + {% for item in inbox_data %} + 
    {{ item |remove_mongo_id|tojson(indent=4) }}
    + {% endfor %} + + {{ utils.display_pagination(older_than, newer_than) }} +
    +
    + +
    +{% endblock %} From 0c60612407b3bac9bdb076691567a3c6e5a4a8d8 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 23 Jul 2018 22:25:51 +0200 Subject: [PATCH 0267/1425] Tweak/debug --- app.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 2b37046..1d45e98 100644 --- a/app.py +++ b/app.py @@ -232,7 +232,7 @@ def _get_file_url(url, size, kind): @app.template_filter() def remove_mongo_id(dat): if "_id" in dat: - del dat["_id"] + dat["_id"] = str(dat["_id"]) return dat @@ -1531,11 +1531,14 @@ def api_undo(): @login_required def admin_stream(): q = {"meta.stream": True, "meta.deleted": False} - inbox_data, older_than, newer_than = paginated_query(DB.activities, q) tpl = "stream.html" if request.args.get("debug"): tpl = "stream_debug.html" + if request.args.get("debug_inbox"): + q = {} + + inbox_data, older_than, newer_than = paginated_query(DB.activities, q) return render_template( tpl, inbox_data=inbox_data, older_than=older_than, newer_than=newer_than From 012b85283ebe268204d0b6ae276226d4409604a8 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 23 Jul 2018 22:34:12 +0200 Subject: [PATCH 0268/1425] Fix the image caching --- tasks.py | 3 ++- templates/utils.html | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tasks.py b/tasks.py index d7f5f05..1670fcf 100644 --- a/tasks.py +++ b/tasks.py @@ -234,7 +234,8 @@ def cache_attachments(self, iri: str) -> None: if activity.has_type(ap.ActivityType.CREATE): for attachment in activity.get_object()._data.get("attachment", []): - MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT) + if attachment.get("mediaType", "").startswith("image/"): + MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT) log.info(f"attachments cached for {iri}") diff --git a/templates/utils.html b/templates/utils.html index ebf5f23..88720f2 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -54,7 +54,7 @@
      {% endif %} {% for a in obj.attachment %} - {% if a.url | get_url | is_img %} + {% if a.mediaType.startswith("image/") %} {% else %}
    • {% if a.filename %}{{ a.filename }}{% else %}{{ a.url }}{% endif %}
      • From f7f2a8994f3a3a847032874234232067282533c3 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 23 Jul 2018 23:50:22 +0200 Subject: [PATCH 0269/1425] Fix the no retry on client error --- tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tasks.py b/tasks.py index 1670fcf..ae525ca 100644 --- a/tasks.py +++ b/tasks.py @@ -151,6 +151,7 @@ def fetch_og_metadata(self, iri: str) -> None: except requests.exceptions.HTTPError as http_err: if 400 <= http_err.response.status_code < 500: log.exception("bad request, no retry") + return log.exception("failed to fetch OG metadata") self.retry( exc=http_err, countdown=int(random.uniform(2, 4) ** self.request.retries) From 937e3e30e3df66ab954b5dbd584ed2df9796e624 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 23 Jul 2018 23:56:22 +0200 Subject: [PATCH 0270/1425] Fix the admin template --- templates/utils.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/utils.html b/templates/utils.html index 88720f2..1e19cac 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -57,7 +57,7 @@ {% if a.mediaType.startswith("image/") %} {% else %} -
    • {% if a.filename %}{{ a.filename }}{% else %}{{ a.url }}{% endif %}
      • +
    • {% if a.filename %}{{ a.filename }}{% else %}{{ a.url }}{% endif %}
      • {% endif %} {% endfor %} {% if obj.attachment | not_only_imgs %} From aa9f35f7ceeefb33420fe1f971af003f369b3d31 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 24 Jul 2018 00:14:35 +0200 Subject: [PATCH 0271/1425] Fix delete replies --- activitypub.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/activitypub.py b/activitypub.py index 4dcb309..7175561 100644 --- a/activitypub.py +++ b/activitypub.py @@ -390,14 +390,22 @@ class MicroblogPubBackend(Backend): ) logger.info(f"inbox_delete handle_replies obj={obj!r}") + in_reply_to = obj.inReplyTo + if delete.get_object().ACTIVITY_TYPE != ap.ActivityType.NOTE: + in_reply_to = DB.activities.find_one( + { + "activity.object.id": delete.get_object().id, + "type": ap.ActivityType.CREATE.value, + } + )["activity"]["object"].get("inReplyTo") # Fake a Undo so any related Like/Announce doesn't appear on the web UI DB.activities.update( {"meta.object.id": obj.id}, {"$set": {"meta.undo": True, "meta.extra": "object deleted"}}, ) - if obj: - self._handle_replies_delete(as_actor, obj) + if in_reply_to: + self._handle_replies_delete(as_actor, in_reply_to) @ensure_it_is_me def outbox_delete(self, as_actor: ap.Person, delete: ap.Delete) -> None: @@ -468,8 +476,9 @@ class MicroblogPubBackend(Backend): self._handle_replies(as_actor, create) @ensure_it_is_me - def _handle_replies_delete(self, as_actor: ap.Person, note: ap.Note) -> None: - in_reply_to = note.inReplyTo + def _handle_replies_delete( + self, as_actor: ap.Person, in_reply_to: Optional[str] + ) -> None: if not in_reply_to: pass From 4910729b2c63700d9062709f5d77593162ef8edc Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 24 Jul 2018 00:25:31 +0200 Subject: [PATCH 0272/1425] Bugfix --- activitypub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitypub.py b/activitypub.py index 7175561..a5fff31 100644 --- a/activitypub.py +++ b/activitypub.py @@ -429,7 +429,7 @@ class MicroblogPubBackend(Backend): {"$set": {"meta.undo": True, "meta.exta": "object deleted"}}, ) - self._handle_replies_delete(as_actor, obj) + self._handle_replies_delete(as_actor, obj.inReplyTo) @ensure_it_is_me def inbox_update(self, as_actor: ap.Person, update: ap.Update) -> None: From cf46ddcb608525089ba7c14afdcac550b9a33167 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 24 Jul 2018 22:10:39 +0200 Subject: [PATCH 0273/1425] Tweak the attachment handling for Hubzilla support --- tasks.py | 7 +++++-- templates/utils.html | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tasks.py b/tasks.py index ae525ca..1d2cf11 100644 --- a/tasks.py +++ b/tasks.py @@ -48,9 +48,9 @@ def process_new_activity(self, iri: str) -> None: tag_stream = False if activity.has_type(ap.ActivityType.ANNOUNCE): - tag_stream = True try: activity.get_object() + tag_stream = True except NotAnActivityError: # Most likely on OStatus notice tag_stream = False @@ -235,7 +235,10 @@ def cache_attachments(self, iri: str) -> None: if activity.has_type(ap.ActivityType.CREATE): for attachment in activity.get_object()._data.get("attachment", []): - if attachment.get("mediaType", "").startswith("image/"): + if ( + attachment.get("mediaType", "").startswith("image/") + or attachment.get("type") == ap.ActivityType.IMAGE.value + ): MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT) log.info(f"attachments cached for {iri}") diff --git a/templates/utils.html b/templates/utils.html index 1e19cac..6a28002 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -54,7 +54,7 @@
        {% endif %} {% for a in obj.attachment %} - {% if a.mediaType.startswith("image/") %} + {% if (a.mediaType and a.mediaType.startswith("image/")) or (a.type and a.type == 'Image') %} {% else %}
      • {% if a.filename %}{{ a.filename }}{% else %}{{ a.url }}{% endif %}
        • From f6dbc41e139bb2d9c686bd2e9eed24ab3ba1a3a6 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 24 Jul 2018 22:45:03 +0200 Subject: [PATCH 0274/1425] Fix the permalink when there's no URL --- templates/utils.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/utils.html b/templates/utils.html index 6a28002..287ab43 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -89,10 +89,10 @@
        {% if perma %}{{ obj.published | format_time }} {% if not (obj.id | is_from_outbox) %} -permalink +permalink {% endif %} {% else %} -permalink +permalink {% endif %} {% if session.logged_in %} From 7e0ff187a223239535beef58b458a1327b1016d8 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 24 Jul 2018 22:47:53 +0200 Subject: [PATCH 0275/1425] Make the per page for the admin stream configurable --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 1d45e98..1ae4424 100644 --- a/app.py +++ b/app.py @@ -1538,7 +1538,7 @@ def admin_stream(): if request.args.get("debug_inbox"): q = {} - inbox_data, older_than, newer_than = paginated_query(DB.activities, q) + inbox_data, older_than, newer_than = paginated_query(DB.activities, q, limit=int(request.args.get('limit', 25))) return render_template( tpl, inbox_data=inbox_data, older_than=older_than, newer_than=newer_than From d544cf893ec01597c9fd00eef2fca7d9c0d5edb9 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 24 Jul 2018 23:44:16 +0200 Subject: [PATCH 0276/1425] Make the pagination debug friendly --- templates/utils.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/utils.html b/templates/utils.html index 287ab43..9eb781b 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -222,10 +222,10 @@ {% macro display_pagination(older_than, newer_than) -%}
        {% if older_than %} - + {% endif %} {% if newer_than %} - + {% endif %}
        {% endmacro -%} From 619044d285938991604e1049985fd181cc92cf84 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 24 Jul 2018 23:58:13 +0200 Subject: [PATCH 0277/1425] Tweak/fix the attachment caching --- app.py | 5 +++-- tasks.py | 5 ++++- utils/media.py | 3 +++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index 1ae4424..e39223f 100644 --- a/app.py +++ b/app.py @@ -225,8 +225,9 @@ def _get_file_url(url, size, kind): _GRIDFS_CACHE[k] = u return u - MEDIA_CACHE.cache(url, kind) - return _get_file_url(url, size, kind) + # MEDIA_CACHE.cache(url, kind) + app.logger.error("cache not available for {url}/{size}/{kind}") + return url @app.template_filter() diff --git a/tasks.py b/tasks.py index 1d2cf11..5bcce31 100644 --- a/tasks.py +++ b/tasks.py @@ -239,7 +239,10 @@ def cache_attachments(self, iri: str) -> None: attachment.get("mediaType", "").startswith("image/") or attachment.get("type") == ap.ActivityType.IMAGE.value ): - MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT) + try: + MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT) + except ValueError: + log.exception(f"failed to cache {attachment}") log.info(f"attachments cached for {iri}") diff --git a/utils/media.py b/utils/media.py index 6767514..66ad02a 100644 --- a/utils/media.py +++ b/utils/media.py @@ -15,6 +15,9 @@ def load(url, user_agent): """Initializes a `PIL.Image` from the URL.""" with requests.get(url, stream=True, headers={"User-Agent": user_agent}) as resp: resp.raise_for_status() + if not resp.headers.get('content-type').startswith('image/'): + raise ValueError(f"bad content-type {resp.headers.get('content-type')}") + resp.raw.decode_content = True return Image.open(BytesIO(resp.raw.read())) From e1374c3148e02ab5fe005a31dd1b19e73bd33e3a Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 25 Jul 2018 00:02:46 +0200 Subject: [PATCH 0278/1425] Tweak the actor --- config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config.py b/config.py index db39fb0..58b4974 100644 --- a/config.py +++ b/config.py @@ -140,6 +140,8 @@ ME = { "summary": SUMMARY, "endpoints": {}, "url": ID, + "manuallyApprovesFollowers": False, + "attachment": [], "icon": { "mediaType": mimetypes.guess_type(ICON_URL)[0], "type": "Image", From 8402bfbed23ab411202a659fa699ef5e12287ec7 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 25 Jul 2018 00:11:23 +0200 Subject: [PATCH 0279/1425] Tweak the admin layout --- templates/header.html | 6 ++---- templates/layout.html | 3 ++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/templates/header.html b/templates/header.html index 287db10..8fc7eb1 100644 --- a/templates/header.html +++ b/templates/header.html @@ -1,3 +1,4 @@ +{% if not request.path.startswith('/admin') %} +{% endif %} diff --git a/templates/layout.html b/templates/layout.html index a61a703..fb7d6d2 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -18,7 +18,8 @@ {% if logged_in %}
          -
        • Admin
          • +
        • Admin
          • +
        • Public
        • New
        • Stream
        • Notifications
          • From ffc2700c4f0627af00e85f5f53af34b05dd8e100 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 26 Jul 2018 22:39:12 +0200 Subject: [PATCH 0280/1425] Tweak/fix the lookup --- utils/lookup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils/lookup.py b/utils/lookup.py index 3bb4756..07627b6 100644 --- a/utils/lookup.py +++ b/utils/lookup.py @@ -15,6 +15,10 @@ def lookup(url: str) -> ap.BaseActivity: return ap.fetch_remote_activity(actor_url) except NotAnActivityError: pass + except requests.HTTPError: + # Some websites may returns 404, 503 or others when they don't support webfinger, and we're just taking a guess + # when performing the lookup. + pass backend = ap.get_backend() resp = requests.get( From d5ee0e3fa5657f49cf3c3340bfabbd1c85ed3f49 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 26 Jul 2018 22:44:31 +0200 Subject: [PATCH 0281/1425] Fix unfollow in the admin --- app.py | 2 +- templates/following.html | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index e39223f..9885a52 100644 --- a/app.py +++ b/app.py @@ -1785,7 +1785,7 @@ def following(): abort(404) following, older_than, newer_than = paginated_query(DB.activities, q) - following = [doc["meta"]["object"] for doc in following] + following = [(doc["remote_id"], doc["meta"]["object"]) for doc in following] return render_template( "following.html", following_data=following, diff --git a/templates/following.html b/templates/following.html index 0acf561..e5874a8 100644 --- a/templates/following.html +++ b/templates/following.html @@ -8,12 +8,12 @@ {% include "header.html" %}
          - {% for follow in following_data %} + {% for (follow_id, follow) in following_data %} {% if session.logged_in %}
          - + From 6d8097d112fb5858cef3e688b5031493b9d5e522 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 26 Jul 2018 22:58:28 +0200 Subject: [PATCH 0282/1425] Add debug mode to help debug thread issues --- app.py | 14 ++++++++++++-- templates/note_debug.html | 23 +++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 templates/note_debug.html diff --git a/app.py b/app.py index 9885a52..41dba64 100644 --- a/app.py +++ b/app.py @@ -232,8 +232,13 @@ def _get_file_url(url, size, kind): @app.template_filter() def remove_mongo_id(dat): + if isinstance(dat, list): + return [remove_mongo_id(item) for item in dat] if "_id" in dat: dat["_id"] = str(dat["_id"]) + for k, v in dat.items(): + if isinstance(v, dict): + dat[k] = remove_mongo_id(dat[k]) return dat @@ -1329,7 +1334,10 @@ def admin_thread(): abort(410) thread = _build_thread(data) - return render_template("note.html", thread=thread, note=data) + tpl = "note.html" + if request.args.get("debug"): + tpl = "note_debug.html" + return render_template(tpl, thread=thread, note=data) @app.route("/admin/new", methods=["GET"]) @@ -1539,7 +1547,9 @@ def admin_stream(): if request.args.get("debug_inbox"): q = {} - inbox_data, older_than, newer_than = paginated_query(DB.activities, q, limit=int(request.args.get('limit', 25))) + inbox_data, older_than, newer_than = paginated_query( + DB.activities, q, limit=int(request.args.get("limit", 25)) + ) return render_template( tpl, inbox_data=inbox_data, older_than=older_than, newer_than=newer_than diff --git a/templates/note_debug.html b/templates/note_debug.html new file mode 100644 index 0000000..af07a4b --- /dev/null +++ b/templates/note_debug.html @@ -0,0 +1,23 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block title %}{{ config.NAME }}: "{{ note.activity.object.content | html2plaintext | truncate(50) }}"{% endblock %} +{% block header %} + + + + + + + + + + +{% endblock %} +{% block content %} +
          +{% include "header.html" %} +
          {{ thread | remove_mongo_id | tojson(indent=4) }}
          +{{ utils.display_thread(thread, likes=likes, shares=shares) }} +
          +{% endblock %} +{% block links %}{% endblock %} From 140d7c5b5bb008d6be78512f3117e035bf8b9728 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 26 Jul 2018 23:04:20 +0200 Subject: [PATCH 0283/1425] Fix debug template --- templates/note_debug.html | 1 - 1 file changed, 1 deletion(-) diff --git a/templates/note_debug.html b/templates/note_debug.html index af07a4b..f34d61c 100644 --- a/templates/note_debug.html +++ b/templates/note_debug.html @@ -17,7 +17,6 @@
          {% include "header.html" %} 
          {{ thread | remove_mongo_id | tojson(indent=4) }}
          -{{ utils.display_thread(thread, likes=likes, shares=shares) }}
          {% endblock %} {% block links %}{% endblock %} From e3b8c4f63cba78edcb7130368640217a8326df20 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 26 Jul 2018 23:11:38 +0200 Subject: [PATCH 0284/1425] Tweak threads for ghost replies --- app.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 41dba64..a2fbd3b 100644 --- a/app.py +++ b/app.py @@ -902,7 +902,10 @@ def _build_thread(data, include_children=True): if rep_id == root_id: continue reply_of = rep["activity"]["object"]["inReplyTo"] - idx[reply_of]["_nodes"].append(rep) + try: + idx[reply_of]["_nodes"].append(rep) + except KeyError: + app.logger.info(f"{reply_of} is not there! skipping {rep}") # Flatten the tree thread = [] From 55ff15ff86340d291af00128177b90734bef0de2 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 29 Jul 2018 16:07:27 +0200 Subject: [PATCH 0285/1425] Move more DB stuff to celery --- activitypub.py | 40 +++++++++--------- app.py | 46 +++++++++------------ tasks.py | 108 ++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 145 insertions(+), 49 deletions(-) diff --git a/activitypub.py b/activitypub.py index a5fff31..470dd84 100644 --- a/activitypub.py +++ b/activitypub.py @@ -1,5 +1,6 @@ import logging import os +import json from datetime import datetime from enum import Enum from typing import Any @@ -120,10 +121,6 @@ class MicroblogPubBackend(Backend): def set_save_cb(self, cb): self.save_cb = cb - @ensure_it_is_me - def outbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: - self.save(Box.OUTBOX, activity) - def followers(self) -> List[str]: q = { "box": Box.INBOX.value, @@ -241,21 +238,9 @@ class MicroblogPubBackend(Backend): def inbox_check_duplicate(self, as_actor: ap.Person, iri: str) -> bool: return bool(DB.activities.find_one({"box": Box.INBOX.value, "remote_id": iri})) - @ensure_it_is_me - def inbox_new(self, as_actor: ap.Person, activity: ap.BaseActivity) -> None: - self.save(Box.INBOX, activity) - def set_post_to_remote_inbox(self, cb): self.post_to_remote_inbox_cb = cb - @ensure_it_is_me - def post_to_remote_inbox(self, as_actor: ap.Person, payload: str, to: str) -> None: - self.post_to_remote_inbox_cb(payload, to) - - @ensure_it_is_me - def new_follower(self, as_actor: ap.Person, follow: ap.Follow) -> None: - pass - @ensure_it_is_me def undo_new_follower(self, as_actor: ap.Person, follow: ap.Follow) -> None: DB.activities.update_one( @@ -268,10 +253,6 @@ class MicroblogPubBackend(Backend): {"remote_id": follow.id}, {"$set": {"meta.undo": True}} ) - @ensure_it_is_me - def new_following(self, as_actor: ap.Person, follow: ap.Follow) -> None: - pass - @ensure_it_is_me def inbox_like(self, as_actor: ap.Person, like: ap.Like) -> None: obj = like.get_object() @@ -527,6 +508,25 @@ class MicroblogPubBackend(Backend): {"$set": {"meta.thread_root_parent": root_reply}}, ) + def post_to_outbox(self, activity: ap.BaseActivity) -> None: + if activity.has_type(ap.CREATE_TYPES): + activity = activity.build_create() + + self.save(Box.OUTBOX, activity) + + # Assign create a random ID + obj_id = self.random_object_id() + activity.set_id(self.activity_url(obj_id), obj_id) + + recipients = activity.recipients() + logger.info(f"recipients={recipients}") + activity = ap.clean_activity(activity.to_dict()) + + payload = json.dumps(activity) + for recp in recipients: + logger.debug(f"posting to {recp}") + self.post_to_remote_inbox(self.get_actor(), payload, recp) + def gen_feed(): fg = FeedGenerator() diff --git a/app.py b/app.py index a2fbd3b..af78472 100644 --- a/app.py +++ b/app.py @@ -74,7 +74,6 @@ from config import PASS from config import USERNAME from config import VERSION from config import _drop_db -from config import custom_cache_purge_hook from utils.key import get_secret_key from utils.lookup import lookup from utils.media import Kind @@ -91,9 +90,6 @@ def save_cb(box: Box, iri: str) -> None: back.set_save_cb(save_cb) - -back.set_post_to_remote_inbox(tasks.post_to_inbox.delay) - ap.use_backend(back) MY_PERSON = ap.Person(**ME) @@ -117,9 +113,6 @@ else: SIG_AUTH = HTTPSigAuth(KEY) -OUTBOX = ap.Outbox(MY_PERSON) -INBOX = ap.Inbox(MY_PERSON) - def verify_pass(pwd): return bcrypt.verify(pwd, PASS) @@ -615,7 +608,7 @@ def authorize_follow(): return redirect("/following") follow = ap.Follow(actor=MY_PERSON.id, object=actor) - OUTBOX.post(follow) + tasks.post_to_outbox(follow) return redirect("/following") @@ -1121,12 +1114,9 @@ def outbox(): data = request.get_json(force=True) print(data) activity = ap.parse_activity(data) - OUTBOX.post(activity) + activity_id = tasks.post_to_outbox(activity) - # Purge the cache if a custom hook is set, as new content was published - custom_cache_purge_hook() - - return Response(status=201, headers={"Location": activity.id}) + return Response(status=201, headers={"Location": activity_id}) @app.route("/outbox/") @@ -1465,9 +1455,9 @@ def api_delete(): note = _user_api_get_note(from_outbox=True) delete = note.build_delete() - OUTBOX.post(delete) + delete_id = tasks.post_to_outbox(delete) - return _user_api_response(activity=delete.id) + return _user_api_response(activity=delete_id) @app.route("/api/boost", methods=["POST"]) @@ -1476,9 +1466,9 @@ def api_boost(): note = _user_api_get_note() announce = note.build_announce(MY_PERSON) - OUTBOX.post(announce) + announce_id = tasks.post_to_outbox(announce) - return _user_api_response(activity=announce.id) + return _user_api_response(activity=announce_id) @app.route("/api/like", methods=["POST"]) @@ -1487,9 +1477,9 @@ def api_like(): note = _user_api_get_note() like = note.build_like(MY_PERSON) - OUTBOX.post(like) + like_id = tasks.post_to_outbox(like) - return _user_api_response(activity=like.id) + return _user_api_response(activity=like_id) @app.route("/api/note/pin", methods=["POST"]) @@ -1534,9 +1524,9 @@ def api_undo(): obj = ap.parse_activity(doc.get("activity")) # FIXME(tsileo): detect already undo-ed and make this API call idempotent undo = obj.build_undo() - OUTBOX.post(undo) + undo_id = tasks.post_to_outbox(undo) - return _user_api_response(activity=undo.id) + return _user_api_response(activity=undo_id) @app.route("/admin/stream") @@ -1605,7 +1595,7 @@ def inbox(): ) activity = ap.parse_activity(data) logger.debug(f"inbox activity={activity}/{data}") - INBOX.post(activity) + tasks.post_to_inbox(activity) return Response(status=201) @@ -1691,9 +1681,9 @@ def api_new_note(): note = ap.Note(**raw_note) create = note.build_create() - OUTBOX.post(create) + create_id = tasks.post_to_outbox(create) - return _user_api_response(activity=create.id) + return _user_api_response(activity=create_id) @app.route("/api/stream") @@ -1724,9 +1714,9 @@ def api_block(): return _user_api_response(activity=existing["activity"]["id"]) block = ap.Block(actor=MY_PERSON.id, object=actor) - OUTBOX.post(block) + block_id = tasks.post_to_outbox(block) - return _user_api_response(activity=block.id) + return _user_api_response(activity=block_id) @app.route("/api/follow", methods=["POST"]) @@ -1746,9 +1736,9 @@ def api_follow(): return _user_api_response(activity=existing["activity"]["id"]) follow = ap.Follow(actor=MY_PERSON.id, object=actor) - OUTBOX.post(follow) + follow_id = tasks.post_to_outbox(follow) - return _user_api_response(activity=follow.id) + return _user_api_response(activity=follow_id) @app.route("/followers") diff --git a/tasks.py b/tasks.py index 5bcce31..6cad65b 100644 --- a/tasks.py +++ b/tasks.py @@ -14,8 +14,10 @@ from little_boxes.linked_data_sig import generate_signature from requests.exceptions import HTTPError import activitypub +from activitypub import Box from config import DB from config import HEADERS +from config import ME from config import ID from config import KEY from config import MEDIA_CACHE @@ -33,6 +35,8 @@ SigAuth = HTTPSigAuth(KEY) back = activitypub.MicroblogPubBackend() ap.use_backend(back) +MY_PERSON = ap.Person(**ME) + @app.task(bind=True, max_retries=12) # noqa: C901 def process_new_activity(self, iri: str) -> None: @@ -253,8 +257,110 @@ def cache_attachments(self, iri: str) -> None: self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) +def post_to_inbox(activity: ap.BaseActivity) -> None: + # Check for Block activity + actor = activity.get_actor() + if back.outbox_is_blocked(MY_PERSON, actor.id): + log.info( + f"actor {actor!r} is blocked, dropping the received activity {activity!r}" + ) + return + + if back.inbox_check_duplicate(MY_PERSON, activity.id): + # The activity is already in the inbox + log.info(f"received duplicate activity {activity!r}, dropping it") + + back.save(Box.INBOX, activity) + finish_post_to_inbox.delay(activity.id) + + +@app.task(bind=True, max_retries=12) # noqa: C901 +def finish_post_to_inbox(self, iri: str) -> None: + try: + activity = ap.fetch_remote_activity(iri) + log.info(f"activity={activity!r}") + + if activity.has_type(ap.ActivityType.DELETE): + back.inbox_delete(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.UPDATE): + back.inbox_update(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.CREATE): + back.inbox_create(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.ANNOUNCE): + back.inbox_announce(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.LIKE): + back.inbox_like(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.FOLLOW): + # Reply to a Follow with an Accept + accept = ap.Accept(actor=ID, object=activity.to_dict(embed=True)) + post_to_outbox(accept) + elif activity.has_type(ap.ActivityType.UNDO): + obj = activity.get_object() + if obj.has_type(ap.ActivityType.LIKE): + back.inbox_undo_like(MY_PERSON, obj) + elif obj.has_type(ap.ActivityType.ANNOUNCE): + back.inbox_undo_announce(MY_PERSON, obj) + elif obj.has_type(ap.ActivityType.FOLLOW): + back.undo_new_follower(MY_PERSON, obj) + + except Exception as err: + log.exception(f"failed to cache attachments for {iri}") + self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) + + +def post_to_outbox(activity: ap.BaseActivity) -> str: + if activity.has_type(ap.CREATE_TYPES): + activity = activity.build_create() + + # Assign create a random ID + obj_id = back.random_object_id() + activity.set_id(back.activity_url(obj_id), obj_id) + + back.save(Box.OUTBOX, activity) + finish_post_to_outbox.delay(activity.id) + return activity.id + + +@app.task(bind=True, max_retries=12) # noqa:C901 +def finish_post_to_outbox(self, iri: str) -> None: + try: + activity = ap.fetch_remote_activity(iri) + log.info(f"activity={activity!r}") + + if activity.has_type(ap.activitytype.delete): + back.outbox_delete(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.UPDATE): + back.outbox_update(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.CREATE): + back.outbox_create(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.ANNOUNCE): + back.outbox_announce(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.LIKE): + back.outbox_like(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.UNDO): + obj = activity.get_object() + if obj.has_type(ap.ActivityType.LIKE): + back.outbox_undo_like(MY_PERSON, obj) + elif obj.has_type(ap.ActivityType.ANNOUNCE): + back.outbox_undo_announce(MY_PERSON, obj) + elif obj.has_type(ap.ActivityType.FOLLOW): + back.undo_new_following(MY_PERSON, obj) + + recipients = activity.recipients() + log.info(f"recipients={recipients}") + activity = ap.clean_activity(activity.to_dict()) + + payload = json.dumps(activity) + for recp in recipients: + log.debug(f"posting to {recp}") + post_to_remote_inbox.delay(payload, recp) + except Exception as err: + log.exception(f"failed to cache attachments for {iri}") + self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) + + @app.task(bind=True, max_retries=12) -def post_to_inbox(self, payload: str, to: str) -> None: +def post_to_remote_inbox(self, payload: str, to: str) -> None: try: log.info("payload=%s", payload) log.info("generating sig") From 5da27d9820731c5075ad84ab03f7ae1cf45026ef Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 29 Jul 2018 16:40:33 +0200 Subject: [PATCH 0286/1425] Bugfixes --- docker-compose-tests.yml | 1 + tasks.py | 13 ++++++++++++- tests/federation_test.py | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docker-compose-tests.yml b/docker-compose-tests.yml index 280f6c3..eae4264 100644 --- a/docker-compose-tests.yml +++ b/docker-compose-tests.yml @@ -26,6 +26,7 @@ services: environment: - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// - MICROBLOGPUB_MONGODB_HOST=mongo:27017 + - MICROBLOGPUB_DEBUG=1 mongo: image: "mongo:latest" rmq: diff --git a/tasks.py b/tasks.py index 6cad65b..6de6575 100644 --- a/tasks.py +++ b/tasks.py @@ -35,6 +35,17 @@ SigAuth = HTTPSigAuth(KEY) back = activitypub.MicroblogPubBackend() ap.use_backend(back) + +def save_cb(box: Box, iri: str) -> None: + if box == Box.INBOX: + process_new_activity.delay(iri) + else: + cache_actor.delay(iri) + + +back.set_save_cb(save_cb) + + MY_PERSON = ap.Person(**ME) @@ -327,7 +338,7 @@ def finish_post_to_outbox(self, iri: str) -> None: activity = ap.fetch_remote_activity(iri) log.info(f"activity={activity!r}") - if activity.has_type(ap.activitytype.delete): + if activity.has_type(ap.ActivityType.DELETE): back.outbox_delete(MY_PERSON, activity) elif activity.has_type(ap.ActivityType.UPDATE): back.outbox_update(MY_PERSON, activity) diff --git a/tests/federation_test.py b/tests/federation_test.py index 20569bb..5ab8e51 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -19,7 +19,7 @@ class Instance(object): def __init__(self, name, host_url, docker_url=None): self.host_url = host_url self.docker_url = docker_url or host_url - self._create_delay = 12 + self._create_delay = 20 with open( os.path.join( os.path.dirname(os.path.abspath(__file__)), From 1d1f8e7cf579ae9ee2f57aac3e282c00f7248b6d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 29 Jul 2018 16:58:25 +0200 Subject: [PATCH 0287/1425] More bugfixes --- tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tasks.py b/tasks.py index 6de6575..09713ea 100644 --- a/tasks.py +++ b/tasks.py @@ -275,13 +275,14 @@ def post_to_inbox(activity: ap.BaseActivity) -> None: log.info( f"actor {actor!r} is blocked, dropping the received activity {activity!r}" ) - return + return if back.inbox_check_duplicate(MY_PERSON, activity.id): # The activity is already in the inbox log.info(f"received duplicate activity {activity!r}, dropping it") back.save(Box.INBOX, activity) + log.info(f"spawning task for {activity!r}") finish_post_to_inbox.delay(activity.id) From efc26fb2a00017ba6e8c7519ea00cc911947e02c Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 29 Jul 2018 17:19:06 +0200 Subject: [PATCH 0288/1425] Fix delete --- app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app.py b/app.py index af78472..bb418ec 100644 --- a/app.py +++ b/app.py @@ -1455,6 +1455,8 @@ def api_delete(): note = _user_api_get_note(from_outbox=True) delete = note.build_delete() + delete = ap.Delete(actor=ID, object=ap.Tombstone(id=note.id).to_dict(embed=True)) + delete_id = tasks.post_to_outbox(delete) return _user_api_response(activity=delete_id) From 7e94d158273a8c52d32c175bb0a8b68ef04cf154 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 29 Jul 2018 17:20:57 +0200 Subject: [PATCH 0289/1425] Bugfix --- app.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app.py b/app.py index bb418ec..8469d22 100644 --- a/app.py +++ b/app.py @@ -1454,7 +1454,6 @@ def api_delete(): """API endpoint to delete a Note activity.""" note = _user_api_get_note(from_outbox=True) - delete = note.build_delete() delete = ap.Delete(actor=ID, object=ap.Tombstone(id=note.id).to_dict(embed=True)) delete_id = tasks.post_to_outbox(delete) From cb0c0a56a79d2945f449de09db7c2d4993513dbb Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 29 Jul 2018 18:23:29 +0200 Subject: [PATCH 0290/1425] Fix the delete --- tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tasks.py b/tasks.py index 09713ea..21b10d3 100644 --- a/tasks.py +++ b/tasks.py @@ -339,6 +339,8 @@ def finish_post_to_outbox(self, iri: str) -> None: activity = ap.fetch_remote_activity(iri) log.info(f"activity={activity!r}") + recipients = activity.recipients() + if activity.has_type(ap.ActivityType.DELETE): back.outbox_delete(MY_PERSON, activity) elif activity.has_type(ap.ActivityType.UPDATE): @@ -358,7 +360,6 @@ def finish_post_to_outbox(self, iri: str) -> None: elif obj.has_type(ap.ActivityType.FOLLOW): back.undo_new_following(MY_PERSON, obj) - recipients = activity.recipients() log.info(f"recipients={recipients}") activity = ap.clean_activity(activity.to_dict()) From e9a4cdf01b02d7a55f4a7d96a420cf46b26649da Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 29 Jul 2018 18:27:59 +0200 Subject: [PATCH 0291/1425] Lower the delay in the federation tests --- tests/federation_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/federation_test.py b/tests/federation_test.py index 5ab8e51..20569bb 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -19,7 +19,7 @@ class Instance(object): def __init__(self, name, host_url, docker_url=None): self.host_url = host_url self.docker_url = docker_url or host_url - self._create_delay = 20 + self._create_delay = 12 with open( os.path.join( os.path.dirname(os.path.abspath(__file__)), From 45bfcc250298946601616635d5b4fb6d1bc511d4 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 29 Jul 2018 19:38:59 +0200 Subject: [PATCH 0292/1425] CSS tweaks --- sass/base_theme.scss | 1 + templates/utils.html | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sass/base_theme.scss b/sass/base_theme.scss index 85447cd..a6748c0 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -229,6 +229,7 @@ a:hover { .note-container { clear: right; padding:10px 0; + word-break: break-all; } } .color-menu-background { diff --git a/templates/utils.html b/templates/utils.html index 9eb781b..5cc6941 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -33,11 +33,11 @@
          - {{ actor.name or actor.preferredUsername }} + {{ actor.name or actor.preferredUsername }} @{% if not no_color and obj.id | is_from_outbox %}{{ actor.preferredUsername }}{% else %}{{ actor.preferredUsername }}{% endif %}@{% if not no_color and obj.id | is_from_outbox %}{{ actor | url_or_id | get_url | domain }}{% else %}{{ actor | url_or_id | get_url | domain }}{% endif %} {% if not perma %} - + From 6cc30f622a447fe65f026bd7886ce10709f24784 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 29 Jul 2018 20:10:15 +0200 Subject: [PATCH 0293/1425] Add Announce support for the lookup --- app.py | 11 ++++++++++- templates/lookup.html | 7 +++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 8469d22..ff9c410 100644 --- a/app.py +++ b/app.py @@ -1303,11 +1303,20 @@ def admin(): @login_required def admin_lookup(): data = None + meta = None if request.method == "POST": if request.form.get("url"): data = lookup(request.form.get("url")) + if data.has_type(ActivityType.ANNOUNCE): + meta = dict( + object=data.get_object().to_dict(), + object_actor=data.get_object().get_actor().to_dict(), + actor=data.get_actor().to_dict(), + ) - return render_template("lookup.html", data=data, url=request.form.get("url")) + return render_template( + "lookup.html", data=data, meta=meta, url=request.form.get("url") + ) @app.route("/admin/thread") diff --git a/templates/lookup.html b/templates/lookup.html index 35af1b8..a8fb5f7 100644 --- a/templates/lookup.html +++ b/templates/lookup.html @@ -31,6 +31,13 @@ {{ utils.display_note(data.object, ui=True) }} {% elif data | has_type('Note') %} {{ utils.display_note(data, ui=True) }} + {% elif data | has_type('Announce') %} + {% set boost_actor = meta.actor %} +

          + {{ boost_actor.name }} boosted +

          + {{ utils.display_note(meta.object, ui=False, meta={'actor': meta.object_actor}) }} + {% endif %}
          {% endif %} From d3db16c63db3df4a0a6c2298d8111f62a2c088d1 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 29 Jul 2018 20:24:46 +0200 Subject: [PATCH 0294/1425] Add support for Article --- app.py | 8 ++++++++ templates/lookup.html | 2 +- templates/utils.html | 11 +++++++++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index ff9c410..ba23e16 100644 --- a/app.py +++ b/app.py @@ -336,6 +336,14 @@ def has_type(doc, _type): return False +@app.template_filter() +def has_actor_type(doc): + for t in ap.ACTOR_TYPES: + if has_type(doc, t.value): + return True + return False + + def _is_img(filename): filename = filename.lower() if ( diff --git a/templates/lookup.html b/templates/lookup.html index a8fb5f7..31932c1 100644 --- a/templates/lookup.html +++ b/templates/lookup.html @@ -15,7 +15,7 @@ {% if data %} {% set data = data.to_dict() %}
          - {% if data | has_type('Person') or data | has_type('Service') %} + {% if data | has_actor_type %}
          diff --git a/templates/utils.html b/templates/utils.html index 5cc6941..78cd27d 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -23,6 +23,7 @@ {% else %} {% set actor = obj.attributedTo | get_actor %} {% endif %} +
          @@ -33,6 +34,7 @@
          + {% if obj.summary %}

          {{ obj.summary | clean }}

          {% endif %}
          + {% if obj | has_type('Article') %} + {{ obj.name }} {{ obj | url_or_id | get_url }} + {% else %} {{ obj.content | clean | safe }} + {% endif %}
          - {% if obj.attachment %} + {% if obj.attachment and obj | has_type('Note') %}
          {% if obj.attachment | not_only_imgs %}

          Attachment

          @@ -68,7 +75,7 @@ -{% if meta and meta.og_metadata %} +{% if meta and meta.og_metadata and obj | has_type('Note') %} {% for og in meta.og_metadata %}
          From 6cda5402f779664f7ba08e43fb67300b29d4b881 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 29 Jul 2018 21:45:17 +0200 Subject: [PATCH 0295/1425] Fix CSS --- sass/base_theme.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sass/base_theme.scss b/sass/base_theme.scss index a6748c0..cdc7665 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -229,7 +229,7 @@ a:hover { .note-container { clear: right; padding:10px 0; - word-break: break-all; + word-break: normal; } } .color-menu-background { From 88186e2306f6ac46c26243be728b5058c044f066 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 30 Jul 2018 09:41:04 +0200 Subject: [PATCH 0296/1425] Tweak the lookup --- app.py | 2 +- templates/utils.html | 2 +- utils/lookup.py | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app.py b/app.py index ba23e16..dce07d3 100644 --- a/app.py +++ b/app.py @@ -219,7 +219,7 @@ def _get_file_url(url, size, kind): return u # MEDIA_CACHE.cache(url, kind) - app.logger.error("cache not available for {url}/{size}/{kind}") + app.logger.error(f"cache not available for {url}/{size}/{kind}") return url diff --git a/templates/utils.html b/templates/utils.html index 78cd27d..8b15a1a 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -40,7 +40,7 @@ {% if not perma %} - + {% endif %} diff --git a/utils/lookup.py b/utils/lookup.py index 07627b6..3e97bfb 100644 --- a/utils/lookup.py +++ b/utils/lookup.py @@ -10,9 +10,10 @@ from little_boxes.webfinger import get_actor_url def lookup(url: str) -> ap.BaseActivity: """Try to find an AP object related to the given URL.""" try: - actor_url = get_actor_url(url) - if actor_url: - return ap.fetch_remote_activity(actor_url) + if url.startswith('@'): + actor_url = get_actor_url(url) + if actor_url: + return ap.fetch_remote_activity(actor_url) except NotAnActivityError: pass except requests.HTTPError: From 9431cb09d72351f2805c79cbb71957a500c024e4 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 30 Jul 2018 10:06:44 +0200 Subject: [PATCH 0297/1425] Fix forwarding --- tasks.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tasks.py b/tasks.py index 21b10d3..a62a1ae 100644 --- a/tasks.py +++ b/tasks.py @@ -114,7 +114,7 @@ def process_new_activity(self, iri: str) -> None: if should_forward: log.info(f"will forward {activity!r} to followers") - activity.forward(back.followers_as_recipients()) + forward_activity.delay(activity.id) if should_delete: log.info(f"will soft delete {activity!r}") @@ -367,6 +367,22 @@ def finish_post_to_outbox(self, iri: str) -> None: for recp in recipients: log.debug(f"posting to {recp}") post_to_remote_inbox.delay(payload, recp) + except Exception as err: + log.exception(f"failed to post to remote inbox for {iri}") + self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) + + +@app.task(bind=True, max_retries=12) # noqa:C901 +def forward_activity(self, iri: str) -> None: + try: + activity = ap.fetch_remote_activity(iri) + recipients = back.followers_as_recipients() + log.debug(f"Forwarding {activity!r} to {recipients}") + activity = ap.clean_activity(activity.to_dict()) + payload = json.dumps(activity) + for recp in recipients: + log.debug(f"forwarding {activity!r} to {recp}") + post_to_remote_inbox.delay(payload, recp) except Exception as err: log.exception(f"failed to cache attachments for {iri}") self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) From cbd9d4e6daae246c2dac5b410399d450ddec87a3 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 30 Jul 2018 18:12:27 +0200 Subject: [PATCH 0298/1425] Show likes in the notifications --- activitypub.py | 10 ---------- app.py | 2 ++ tasks.py | 27 +++++++++++++++++++++++++++ templates/stream.html | 9 +++++++++ 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/activitypub.py b/activitypub.py index 470dd84..ff75354 100644 --- a/activitypub.py +++ b/activitypub.py @@ -280,16 +280,6 @@ class MicroblogPubBackend(Backend): {"$inc": {"meta.count_like": 1}, "$set": {"meta.liked": like.id}}, ) - DB.activities.update_one( - {"remote_id": like.id}, - { - "$set": { - "meta.object": obj.to_dict(embed=True), - "meta.object_actor": _actor_to_meta(obj.get_actor()), - } - }, - ) - @ensure_it_is_me def outbox_undo_like(self, as_actor: ap.Person, like: ap.Like) -> None: obj = like.get_object() diff --git a/app.py b/app.py index dce07d3..c2f32c3 100644 --- a/app.py +++ b/app.py @@ -1397,6 +1397,7 @@ def admin_notifications(): "type": ActivityType.UNDO.value, "activity.object.type": ActivityType.FOLLOW.value, } + likes_query = {"type": ActivityType.LIKE.value} followed_query = {"type": ActivityType.ACCEPT.value} q = { "box": Box.INBOX.value, @@ -1407,6 +1408,7 @@ def admin_notifications(): new_followers_query, followed_query, unfollow_query, + likes_query, ], } inbox_data, older_than, newer_than = paginated_query(DB.activities, q) diff --git a/tasks.py b/tasks.py index a62a1ae..944b464 100644 --- a/tasks.py +++ b/tasks.py @@ -176,6 +176,30 @@ def fetch_og_metadata(self, iri: str) -> None: self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) +@app.task(bind=True, max_retries=12) +def cache_object(self, iri: str) -> None: + try: + activity = ap.fetch_remote_activity(iri) + log.info(f"activity={activity!r}") + + obj = activity.get_object() + DB.activities.update_one( + {"remote_id": activity.id}, + { + "$set": { + "meta.object": obj.to_dict(embed=True), + "meta.object_actor": activitypub._actor_to_meta(obj.get_actor()), + } + }, + ) + except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): + DB.activities.update_one({"remote_id": iri}, {"$set": {"meta.deleted": True}}) + log.exception(f"flagging activity {iri} as deleted, no object caching") + except Exception as err: + log.exception(f"failed to cache object for {iri}") + self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) + + @app.task(bind=True, max_retries=12) def cache_actor(self, iri: str, also_cache_attachments: bool = True) -> None: try: @@ -185,6 +209,9 @@ def cache_actor(self, iri: str, also_cache_attachments: bool = True) -> None: if activity.has_type(ap.ActivityType.CREATE): fetch_og_metadata.delay(iri) + if activity.has_type([ap.ActivityType.LIKE, ap.ActivityType.ANNOUNCE]): + cache_object.delay(iri) + actor = activity.get_actor() cache_actor_with_inbox = False diff --git a/templates/stream.html b/templates/stream.html index f12c0c7..d9af1c2 100644 --- a/templates/stream.html +++ b/templates/stream.html @@ -19,6 +19,15 @@ {{ utils.display_note(item.meta.object, ui=True) }} {% endif %} {% endif %} + + {% if item | has_type('Like') %} + {% set boost_actor = item.meta.actor %} +

          {{ boost_actor.name or boost_actor.preferredUsername }} liked

          + {{ item }} + {% if item.meta.object %} + {{ utils.display_note(item.meta.object, ui=False, meta={'actor': item.meta.object_actor}) }} + {% endif %} + {% endif %} {% if item | has_type('Follow') %}

          new follower From 844a65e9a58bafac374321a5587bd9da5882a876 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 30 Jul 2018 18:13:19 +0200 Subject: [PATCH 0299/1425] Fix debug --- templates/stream.html | 1 - 1 file changed, 1 deletion(-) diff --git a/templates/stream.html b/templates/stream.html index d9af1c2..ca22897 100644 --- a/templates/stream.html +++ b/templates/stream.html @@ -23,7 +23,6 @@ {% if item | has_type('Like') %} {% set boost_actor = item.meta.actor %}

          {{ boost_actor.name or boost_actor.preferredUsername }} liked

          - {{ item }} {% if item.meta.object %} {{ utils.display_note(item.meta.object, ui=False, meta={'actor': item.meta.object_actor}) }} {% endif %} From ff7211ae72d1a57fa665ab101a247c8c73a711c0 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 30 Jul 2018 18:30:47 +0200 Subject: [PATCH 0300/1425] Tweak the like query --- app.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index c2f32c3..c0fcd57 100644 --- a/app.py +++ b/app.py @@ -1397,7 +1397,10 @@ def admin_notifications(): "type": ActivityType.UNDO.value, "activity.object.type": ActivityType.FOLLOW.value, } - likes_query = {"type": ActivityType.LIKE.value} + likes_query = { + "type": ActivityType.LIKE.value, + "activity.object": {"$regex": f"^{BASE_URL}"}, + } followed_query = {"type": ActivityType.ACCEPT.value} q = { "box": Box.INBOX.value, From 8f3208175cfb97d5347e09a326c41fcaa6cf37af Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 31 Jul 2018 22:42:50 +0200 Subject: [PATCH 0301/1425] Allow to reply to external activities --- app.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index c0fcd57..0899332 100644 --- a/app.py +++ b/app.py @@ -1356,12 +1356,20 @@ def admin_new(): reply_id = None content = "" thread = [] + print(request.args) if request.args.get("reply"): data = DB.activities.find_one({"activity.object.id": request.args.get("reply")}) - if not data: - abort(400) + if data: + reply = ap.parse_activity(data["activity"]) + else: + data = dict( + meta={}, + activity=dict( + object=get_backend().fetch_iri(request.args.get("reply")) + ), + ) + reply = ap.parse_activity(data["activity"]["object"]) - reply = ap.parse_activity(data["activity"]) reply_id = reply.id if reply.ACTIVITY_TYPE == ActivityType.CREATE: reply_id = reply.get_object().id From 29a2cc23ffdcb21500d5b13d0e5d55fafb6d0330 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 31 Jul 2018 23:23:20 +0200 Subject: [PATCH 0302/1425] Fix pagination --- app.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app.py b/app.py index 0899332..221bcf6 100644 --- a/app.py +++ b/app.py @@ -1110,6 +1110,7 @@ def outbox(): q=q, cursor=request.args.get("cursor"), map_func=lambda doc: activity_from_doc(doc, embed=True), + col_name="outbox", ) ) @@ -1596,6 +1597,7 @@ def inbox(): q={"meta.deleted": False, "box": Box.INBOX.value}, cursor=request.args.get("cursor"), map_func=lambda doc: remove_context(doc["activity"]), + col_name="inbox", ) ) @@ -1783,6 +1785,7 @@ def followers(): q=q, cursor=request.args.get("cursor"), map_func=lambda doc: doc["activity"]["actor"], + col_name="followers", ) ) @@ -1812,6 +1815,7 @@ def following(): q=q, cursor=request.args.get("cursor"), map_func=lambda doc: doc["activity"]["object"], + col_name="following", ) ) From 53e5a9e23714f15fc4ebcbc18c0e88beaccb3ca2 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 1 Aug 2018 08:29:08 +0200 Subject: [PATCH 0303/1425] Add debug logs --- app.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app.py b/app.py index 221bcf6..9a31af0 100644 --- a/app.py +++ b/app.py @@ -939,6 +939,7 @@ def note_by_id(note_id): if data["meta"].get("deleted", False): abort(410) thread = _build_thread(data) + app.logger.info(f"thread={thread!r}") likes = list( DB.activities.find( @@ -947,6 +948,7 @@ def note_by_id(note_id): "meta.deleted": False, "type": ActivityType.LIKE.value, "$or": [ + # FIXME(tsileo): remove all the useless $or {"activity.object.id": data["activity"]["object"]["id"]}, {"activity.object": data["activity"]["object"]["id"]}, ], @@ -954,6 +956,7 @@ def note_by_id(note_id): ) ) likes = [doc["meta"]["actor"] for doc in likes] + app.logger.info(f"likes={likes!r}") shares = list( DB.activities.find( @@ -969,6 +972,7 @@ def note_by_id(note_id): ) ) shares = [doc["meta"]["actor"] for doc in shares] + app.logger.info(f"shares={shares!r}") return render_template( "note.html", likes=likes, shares=shares, thread=thread, note=data From 5dce02570049cdd303c4d9367fe5dca120c64091 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 5 Aug 2018 13:49:49 +0200 Subject: [PATCH 0304/1425] Tasks cleanup --- activitypub.py | 5 ----- app.py | 11 ----------- tasks.py | 27 +++++++++++++++------------ 3 files changed, 15 insertions(+), 28 deletions(-) diff --git a/activitypub.py b/activitypub.py index ff75354..6aa3be5 100644 --- a/activitypub.py +++ b/activitypub.py @@ -116,11 +116,6 @@ class MicroblogPubBackend(Backend): } ) - self.save_cb(box, activity.id) - - def set_save_cb(self, cb): - self.save_cb = cb - def followers(self) -> List[str]: q = { "box": Box.INBOX.value, diff --git a/app.py b/app.py index 9a31af0..81cc10b 100644 --- a/app.py +++ b/app.py @@ -79,17 +79,6 @@ from utils.lookup import lookup from utils.media import Kind back = activitypub.MicroblogPubBackend() - - -def save_cb(box: Box, iri: str) -> None: - if box == Box.INBOX: - tasks.process_new_activity.delay(iri) - else: - tasks.cache_actor.delay(iri) - - -back.set_save_cb(save_cb) - ap.use_backend(back) MY_PERSON = ap.Person(**ME) diff --git a/tasks.py b/tasks.py index 944b464..0cf24e4 100644 --- a/tasks.py +++ b/tasks.py @@ -22,6 +22,7 @@ from config import ID from config import KEY from config import MEDIA_CACHE from config import USER_AGENT +from config import BASE_URL from utils import opengraph from utils.media import Kind @@ -36,16 +37,6 @@ back = activitypub.MicroblogPubBackend() ap.use_backend(back) -def save_cb(box: Box, iri: str) -> None: - if box == Box.INBOX: - process_new_activity.delay(iri) - else: - cache_actor.delay(iri) - - -back.set_save_cb(save_cb) - - MY_PERSON = ap.Person(**ME) @@ -70,6 +61,9 @@ def process_new_activity(self, iri: str) -> None: # Most likely on OStatus notice tag_stream = False should_delete = True + except (ActivityGoneError, ActivityNotFoundError): + # The announced activity is deleted/gone, drop it + should_delete = True elif activity.has_type(ap.ActivityType.CREATE): note = activity.get_object() @@ -112,6 +106,12 @@ def process_new_activity(self, iri: str) -> None: # If the activity was originally forwarded, forward the delete too should_forward = True + elif activity.has_type(ap.ActivityType.LIKE): + if not activity.get_object_id.startswith(BASE_URL): + # We only want to keep a like if it's a like for a local activity + # (Pleroma relay the likes it received, we don't want to store them) + should_delete = True + if should_forward: log.info(f"will forward {activity!r} to followers") forward_activity.delay(activity.id) @@ -132,7 +132,7 @@ def process_new_activity(self, iri: str) -> None: ) log.info(f"new activity {iri} processed") - if not should_delete: + if not should_delete and not activity.has_type(ap.ActivityType.DELETE): cache_actor.delay(iri) except (ActivityGoneError, ActivityNotFoundError): log.exception(f"dropping activity {iri}, skip processing") @@ -245,7 +245,7 @@ def cache_actor(self, iri: str, also_cache_attachments: bool = True) -> None: ) log.info(f"actor cached for {iri}") - if also_cache_attachments: + if also_cache_attachments and activity.has_type(ap.ActivityType.CREATE): cache_attachments.delay(iri) except (ActivityGoneError, ActivityNotFoundError): @@ -309,6 +309,8 @@ def post_to_inbox(activity: ap.BaseActivity) -> None: log.info(f"received duplicate activity {activity!r}, dropping it") back.save(Box.INBOX, activity) + process_new_activity.delay(activity.id) + log.info(f"spawning task for {activity!r}") finish_post_to_inbox.delay(activity.id) @@ -356,6 +358,7 @@ def post_to_outbox(activity: ap.BaseActivity) -> str: activity.set_id(back.activity_url(obj_id), obj_id) back.save(Box.OUTBOX, activity) + cache_actor.delay(activity.id) finish_post_to_outbox.delay(activity.id) return activity.id From 34579bd1518a5d3859ed97706b50ae494f932d0b Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 5 Aug 2018 13:55:48 +0200 Subject: [PATCH 0305/1425] Tweak OpenGraph --- templates/utils.html | 2 ++ utils/opengraph.py | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/templates/utils.html b/templates/utils.html index 8b15a1a..3659366 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -77,6 +77,7 @@ {% if meta and meta.og_metadata and obj | has_type('Note') %} {% for og in meta.og_metadata %} +{% if og.url %}
          @@ -87,6 +88,7 @@ {{ og.site_name }}
          +{% endif %} {% endfor %} {% endif %} diff --git a/utils/opengraph.py b/utils/opengraph.py index 87ffa5e..30c752b 100644 --- a/utils/opengraph.py +++ b/utils/opengraph.py @@ -42,4 +42,11 @@ def fetch_og_metadata(user_agent, links): r = requests.get(l, headers={"User-Agent": user_agent}, timeout=15) r.raise_for_status() htmls.append(r.text) - return [dict(opengraph.OpenGraph(html=html)) for html in htmls] + + res = [] + for html in htmls: + data = dict(opengraph.OpenGraph(html=html)) + if data.get("url"): + res.append(data) + + return res From 57c6c765b60be94ac9d98ed5f7a9fe932823ce1e Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 5 Aug 2018 14:04:09 +0200 Subject: [PATCH 0306/1425] Tweak the tasks --- tasks.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tasks.py b/tasks.py index 0cf24e4..15c27a7 100644 --- a/tasks.py +++ b/tasks.py @@ -320,7 +320,10 @@ def finish_post_to_inbox(self, iri: str) -> None: try: activity = ap.fetch_remote_activity(iri) log.info(f"activity={activity!r}") + except (ActivityGoneError, ActivityNotFoundError): + log.exception(f"no retry") + try: if activity.has_type(ap.ActivityType.DELETE): back.inbox_delete(MY_PERSON, activity) elif activity.has_type(ap.ActivityType.UPDATE): @@ -343,7 +346,6 @@ def finish_post_to_inbox(self, iri: str) -> None: back.inbox_undo_announce(MY_PERSON, obj) elif obj.has_type(ap.ActivityType.FOLLOW): back.undo_new_follower(MY_PERSON, obj) - except Exception as err: log.exception(f"failed to cache attachments for {iri}") self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) @@ -368,7 +370,10 @@ def finish_post_to_outbox(self, iri: str) -> None: try: activity = ap.fetch_remote_activity(iri) log.info(f"activity={activity!r}") + except (ActivityGoneError, ActivityNotFoundError): + log.exception(f"no retry") + try: recipients = activity.recipients() if activity.has_type(ap.ActivityType.DELETE): From b43fa4556e729728cf4e173c73d782a4f53370bf Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 5 Aug 2018 14:21:01 +0200 Subject: [PATCH 0307/1425] Fix the retry --- tasks.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tasks.py b/tasks.py index 15c27a7..d1c3356 100644 --- a/tasks.py +++ b/tasks.py @@ -320,10 +320,7 @@ def finish_post_to_inbox(self, iri: str) -> None: try: activity = ap.fetch_remote_activity(iri) log.info(f"activity={activity!r}") - except (ActivityGoneError, ActivityNotFoundError): - log.exception(f"no retry") - try: if activity.has_type(ap.ActivityType.DELETE): back.inbox_delete(MY_PERSON, activity) elif activity.has_type(ap.ActivityType.UPDATE): @@ -346,6 +343,8 @@ def finish_post_to_inbox(self, iri: str) -> None: back.inbox_undo_announce(MY_PERSON, obj) elif obj.has_type(ap.ActivityType.FOLLOW): back.undo_new_follower(MY_PERSON, obj) + except (ActivityGoneError, ActivityNotFoundError): + log.exception(f"no retry") except Exception as err: log.exception(f"failed to cache attachments for {iri}") self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) @@ -370,10 +369,7 @@ def finish_post_to_outbox(self, iri: str) -> None: try: activity = ap.fetch_remote_activity(iri) log.info(f"activity={activity!r}") - except (ActivityGoneError, ActivityNotFoundError): - log.exception(f"no retry") - try: recipients = activity.recipients() if activity.has_type(ap.ActivityType.DELETE): @@ -402,6 +398,8 @@ def finish_post_to_outbox(self, iri: str) -> None: for recp in recipients: log.debug(f"posting to {recp}") post_to_remote_inbox.delay(payload, recp) + except (ActivityGoneError, ActivityNotFoundError): + log.exception(f"no retry") except Exception as err: log.exception(f"failed to post to remote inbox for {iri}") self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) From c585f0785726fd01d758f36f826169f6a0946019 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 5 Aug 2018 14:24:52 +0200 Subject: [PATCH 0308/1425] More opengraph tweaks --- utils/opengraph.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/utils/opengraph.py b/utils/opengraph.py index 30c752b..5401131 100644 --- a/utils/opengraph.py +++ b/utils/opengraph.py @@ -1,3 +1,4 @@ +import logging import opengraph import requests from bs4 import BeautifulSoup @@ -8,6 +9,8 @@ from little_boxes.urlutils import is_url_valid from .lookup import lookup +logger = logging.getLogger(__name__) + def links_from_note(note): tags_href = set() @@ -27,7 +30,7 @@ def links_from_note(note): def fetch_og_metadata(user_agent, links): - htmls = [] + res = [] for l in links: check_url(l) @@ -41,11 +44,13 @@ def fetch_og_metadata(user_agent, links): r = requests.get(l, headers={"User-Agent": user_agent}, timeout=15) r.raise_for_status() - htmls.append(r.text) - res = [] - for html in htmls: - data = dict(opengraph.OpenGraph(html=html)) + html = r.text + try: + data = dict(opengraph.OpenGraph(html=html)) + except Exception: + logger.exception("failed to parse {l}") + continue if data.get("url"): res.append(data) From 3d8d3efd25b6a0ba14cf1794da92f2c40b13a040 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 5 Aug 2018 14:45:44 +0200 Subject: [PATCH 0309/1425] More Open Graph tweaks --- utils/opengraph.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils/opengraph.py b/utils/opengraph.py index 5401131..318b1bc 100644 --- a/utils/opengraph.py +++ b/utils/opengraph.py @@ -44,12 +44,15 @@ def fetch_og_metadata(user_agent, links): r = requests.get(l, headers={"User-Agent": user_agent}, timeout=15) r.raise_for_status() + if not r.headers.get("content-type").startswith("text/html"): + logger.debug(f"skipping {l}") + continue html = r.text try: data = dict(opengraph.OpenGraph(html=html)) except Exception: - logger.exception("failed to parse {l}") + logger.exception(f"failed to parse {l}") continue if data.get("url"): res.append(data) From 7aff43f0f60378d3b5bea74f3f93430d661ac5b6 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 5 Aug 2018 19:31:47 +0200 Subject: [PATCH 0310/1425] Bugfix --- tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks.py b/tasks.py index d1c3356..dfc7644 100644 --- a/tasks.py +++ b/tasks.py @@ -107,7 +107,7 @@ def process_new_activity(self, iri: str) -> None: should_forward = True elif activity.has_type(ap.ActivityType.LIKE): - if not activity.get_object_id.startswith(BASE_URL): + if not activity.get_object_id().startswith(BASE_URL): # We only want to keep a like if it's a like for a local activity # (Pleroma relay the likes it received, we don't want to store them) should_delete = True From a8baa88fb59e4c5a45ba4478dab634a1233cc9e5 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 6 Aug 2018 09:42:32 +0200 Subject: [PATCH 0311/1425] Tweak the background tasks retry --- tasks.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/tasks.py b/tasks.py index dfc7644..c05d200 100644 --- a/tasks.py +++ b/tasks.py @@ -39,8 +39,10 @@ ap.use_backend(back) MY_PERSON = ap.Person(**ME) +MAX_RETRIES = 9 -@app.task(bind=True, max_retries=12) # noqa: C901 + +@app.task(bind=True, max_retries=MAX_RETRIES) # noqa: C901 def process_new_activity(self, iri: str) -> None: """Process an activity received in the inbox""" try: @@ -141,7 +143,7 @@ def process_new_activity(self, iri: str) -> None: self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) -@app.task(bind=True, max_retries=12) # noqa: C901 +@app.task(bind=True, max_retries=MAX_RETRIES) # noqa: C901 def fetch_og_metadata(self, iri: str) -> None: try: activity = ap.fetch_remote_activity(iri) @@ -176,7 +178,7 @@ def fetch_og_metadata(self, iri: str) -> None: self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) -@app.task(bind=True, max_retries=12) +@app.task(bind=True, max_retries=MAX_RETRIES) def cache_object(self, iri: str) -> None: try: activity = ap.fetch_remote_activity(iri) @@ -200,7 +202,7 @@ def cache_object(self, iri: str) -> None: self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) -@app.task(bind=True, max_retries=12) +@app.task(bind=True, max_retries=MAX_RETRIES) def cache_actor(self, iri: str, also_cache_attachments: bool = True) -> None: try: activity = ap.fetch_remote_activity(iri) @@ -256,7 +258,7 @@ def cache_actor(self, iri: str, also_cache_attachments: bool = True) -> None: self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) -@app.task(bind=True, max_retries=12) +@app.task(bind=True, max_retries=MAX_RETRIES) def cache_attachments(self, iri: str) -> None: try: activity = ap.fetch_remote_activity(iri) @@ -288,7 +290,7 @@ def cache_attachments(self, iri: str) -> None: log.info(f"attachments cached for {iri}") - except (ActivityGoneError, ActivityNotFoundError): + except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): log.exception(f"dropping activity {iri}, no attachment caching") except Exception as err: log.exception(f"failed to cache attachments for {iri}") @@ -315,7 +317,7 @@ def post_to_inbox(activity: ap.BaseActivity) -> None: finish_post_to_inbox.delay(activity.id) -@app.task(bind=True, max_retries=12) # noqa: C901 +@app.task(bind=True, max_retries=MAX_RETRIES) # noqa: C901 def finish_post_to_inbox(self, iri: str) -> None: try: activity = ap.fetch_remote_activity(iri) @@ -343,7 +345,7 @@ def finish_post_to_inbox(self, iri: str) -> None: back.inbox_undo_announce(MY_PERSON, obj) elif obj.has_type(ap.ActivityType.FOLLOW): back.undo_new_follower(MY_PERSON, obj) - except (ActivityGoneError, ActivityNotFoundError): + except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): log.exception(f"no retry") except Exception as err: log.exception(f"failed to cache attachments for {iri}") @@ -364,7 +366,7 @@ def post_to_outbox(activity: ap.BaseActivity) -> str: return activity.id -@app.task(bind=True, max_retries=12) # noqa:C901 +@app.task(bind=True, max_retries=MAX_RETRIES) # noqa:C901 def finish_post_to_outbox(self, iri: str) -> None: try: activity = ap.fetch_remote_activity(iri) @@ -405,7 +407,7 @@ def finish_post_to_outbox(self, iri: str) -> None: self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) -@app.task(bind=True, max_retries=12) # noqa:C901 +@app.task(bind=True, max_retries=MAX_RETRIES) # noqa:C901 def forward_activity(self, iri: str) -> None: try: activity = ap.fetch_remote_activity(iri) @@ -421,7 +423,7 @@ def forward_activity(self, iri: str) -> None: self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries)) -@app.task(bind=True, max_retries=12) +@app.task(bind=True, max_retries=MAX_RETRIES) def post_to_remote_inbox(self, payload: str, to: str) -> None: try: log.info("payload=%s", payload) From 2464dd8782eee316767682b2edceec089fed6f92 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 28 Aug 2018 22:14:48 +0200 Subject: [PATCH 0312/1425] Fixes invalid likes/boost --- app.py | 18 ++++++++++++++---- templates/stream.html | 2 ++ templates/utils.html | 4 ++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/app.py b/app.py index 81cc10b..76edaf1 100644 --- a/app.py +++ b/app.py @@ -930,7 +930,7 @@ def note_by_id(note_id): thread = _build_thread(data) app.logger.info(f"thread={thread!r}") - likes = list( + raw_likes = list( DB.activities.find( { "meta.undo": False, @@ -944,10 +944,15 @@ def note_by_id(note_id): } ) ) - likes = [doc["meta"]["actor"] for doc in likes] + likes = [] + for doc in raw_likes: + try: + likes.append(doc["meta"]["actor"]) + except Exception: + app.logger.exception(f"invalid doc: {doc!r}") app.logger.info(f"likes={likes!r}") - shares = list( + raw_shares = list( DB.activities.find( { "meta.undo": False, @@ -960,7 +965,12 @@ def note_by_id(note_id): } ) ) - shares = [doc["meta"]["actor"] for doc in shares] + shares = [] + for doc in raw_shares: + try: + shares.append(doc["meta"]["actor"]) + except Exception: + app.logger.exception(f"invalid doc: {doc!r}") app.logger.info(f"shares={shares!r}") return render_template( diff --git a/templates/stream.html b/templates/stream.html index ca22897..37d1394 100644 --- a/templates/stream.html +++ b/templates/stream.html @@ -14,7 +14,9 @@ {% if item | has_type('Announce') %} {% set boost_actor = item.meta.actor %} + {% if boost_actor %}

          {{ boost_actor.name or boost_actor.preferredUsername }} boosted

          + {% endif %} {% if item.meta.object %} {{ utils.display_note(item.meta.object, ui=True) }} {% endif %} diff --git a/templates/utils.html b/templates/utils.html index 3659366..6dc11e1 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -195,14 +195,14 @@
          {% if likes %}
          -

          {{ meta.count_like }} likes

          {% for like in likes %} +

          {{ likes|length }} likes

          {% for like in likes %} {{ display_actor_inline(like) }} {% endfor %}
          {% endif %} {% if shares %}
          -

          {{ meta.count_boost }} boosts

          {% for boost in shares %} +

          {{ shares|length }} boosts

          {% for boost in shares %} {{ display_actor_inline(boost) }} {% endfor %}
          From 269c5135c21c03600a177bdef4a9ff8beef0eb1f Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 1 Sep 2018 11:17:46 +0200 Subject: [PATCH 0313/1425] Start working on indexes --- Dockerfile | 1 + config.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/Dockerfile b/Dockerfile index 8202116..e813f1c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,4 +3,5 @@ ADD . /app WORKDIR /app RUN pip install -r requirements.txt ENV FLASK_APP=app.py +RUN python -c "import config; config.create_indexes()" CMD ["gunicorn", "-t", "300", "-w", "2", "-b", "0.0.0.0:5005", "--log-level", "debug", "app:app"] diff --git a/config.py b/config.py index 58b4974..f6af26d 100644 --- a/config.py +++ b/config.py @@ -11,6 +11,7 @@ from itsdangerous import JSONWebSignatureSerializer from little_boxes import strtobool from little_boxes.activitypub import DEFAULT_CTX from pymongo import MongoClient +import pymongo from utils.key import KEY_DIR from utils.key import get_key @@ -101,6 +102,38 @@ GRIDFS = mongo_client[f"{DB_NAME}_gridfs"] MEDIA_CACHE = MediaCache(GRIDFS, USER_AGENT) +def create_indexes(): + DB.activities.create_index([("remote_id", pymongo.ASCENDING)]) + + # Index for the block query + DB.activities.create_index( + [ + ("box", pymongo.ASCENDING), + ("type", pymongo.ASCENDING), + ("meta.undo", pymongo.ASCENDING), + ] + ) + + # Index for count queries + DB.activities.create_index( + [ + ("box", pymongo.ASCENDING), + ("type", pymongo.ASCENDING), + ("meta.undo", pymongo.ASCENDING), + ("meta.deleted", pymongo.ASCENDING), + ] + ) + + DB.activities.create_index( + [ + ("type", pymongo.ASCENDING), + ("activity.object.type", pymongo.ASCENDING), + ("activity.object.inReplyTo", pymongo.ASCENDING), + ("meta.deleted", pymongo.ASCENDING), + ] + ) + + def _drop_db(): if not DEBUG_MODE: return From f6586eb8529107fbaa02b3f332919c6d58f2d5fc Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 1 Sep 2018 11:23:19 +0200 Subject: [PATCH 0314/1425] Tweak the Dockerfile --- Dockerfile | 3 +-- run.sh | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100755 run.sh diff --git a/Dockerfile b/Dockerfile index e813f1c..5ce7e72 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,5 +3,4 @@ ADD . /app WORKDIR /app RUN pip install -r requirements.txt ENV FLASK_APP=app.py -RUN python -c "import config; config.create_indexes()" -CMD ["gunicorn", "-t", "300", "-w", "2", "-b", "0.0.0.0:5005", "--log-level", "debug", "app:app"] +CMD ["./run.sh"] diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..a29f5fc --- /dev/null +++ b/run.sh @@ -0,0 +1,3 @@ +#!/bin/bash +python -c "import config; config.create_indexes()" +gunicorn -t 300 -w 2 -b 0.0.0.0:5005 --log-level debug app:app From f17d625a39ad8538e53dc1244a3f46d86b28750e Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 1 Sep 2018 11:38:54 +0200 Subject: [PATCH 0315/1425] Tweak the indexes --- config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/config.py b/config.py index f6af26d..a31aeca 100644 --- a/config.py +++ b/config.py @@ -104,6 +104,7 @@ MEDIA_CACHE = MediaCache(GRIDFS, USER_AGENT) def create_indexes(): DB.activities.create_index([("remote_id", pymongo.ASCENDING)]) + DB.activities.create_index([("activity.object.id", pymongo.ASCENDING)]) # Index for the block query DB.activities.create_index( From bd0eaf5252aadabad43de413edd9d8e9f0926f29 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 1 Sep 2018 12:54:39 +0200 Subject: [PATCH 0316/1425] Fix CI --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 8540f59..5094276 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,7 @@ script: - docker-compose -p instance1 -f docker-compose-tests.yml ps - WEB_PORT=5007 CONFIG_DIR=./tests/fixtures/instance2/config docker-compose -p instance2 -f docker-compose-tests.yml up -d - docker-compose -p instance2 -f docker-compose-tests.yml ps + - sleep 5 - curl http://localhost:5006 - curl http://localhost:5007 # Integration tests first From 3b186f23d3908881196ff85fcbe516d129ad215b Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 2 Sep 2018 19:43:09 +0200 Subject: [PATCH 0317/1425] Video support --- app.py | 29 ++++++++++++++++++++++++----- config.py | 4 ++++ sass/base_theme.scss | 3 +++ templates/lookup.html | 2 +- templates/utils.html | 15 ++++++++++++--- 5 files changed, 44 insertions(+), 9 deletions(-) diff --git a/app.py b/app.py index 76edaf1..212702e 100644 --- a/app.py +++ b/app.py @@ -224,6 +224,14 @@ def remove_mongo_id(dat): return dat +@app.template_filter() +def get_video_link(data): + for link in data: + if link.get("mimeType", "").startswith("video/"): + return link.get("href") + return None + + @app.template_filter() def get_actor_icon_url(url, size): return _get_file_url(url, size, Kind.ACTOR_ICON) @@ -281,6 +289,11 @@ def url_or_id(d): @app.template_filter() def get_url(u): + print(f'GET_URL({u!r})') + if isinstance(u, list): + for l in u: + if l.get('mimeType') == 'text/html': + u = l if isinstance(u, dict): return u["href"] elif isinstance(u, str): @@ -293,13 +306,17 @@ def get_url(u): def get_actor(url): if not url: return None + if isinstance(url, list): + url = url[0] + if isinstance(url, dict): + url = url.get("id") print(f"GET_ACTOR {url}") try: return get_backend().fetch_iri(url) except (ActivityNotFoundError, ActivityGoneError): return f"Deleted<{url}>" - except Exception: - return f"Error<{url}>" + except Exception as exc: + return f"Error<{url}/{exc!r}>" @app.template_filter() @@ -319,9 +336,10 @@ def format_timeago(val): @app.template_filter() -def has_type(doc, _type): - if _type in _to_list(doc["type"]): - return True +def has_type(doc, _types): + for _type in _to_list(_types): + if _type in _to_list(doc["type"]): + return True return False @@ -1326,6 +1344,7 @@ def admin_lookup(): actor=data.get_actor().to_dict(), ) + print(data) return render_template( "lookup.html", data=data, meta=meta, url=request.form.get("url") ) diff --git a/config.py b/config.py index a31aeca..a5940ce 100644 --- a/config.py +++ b/config.py @@ -105,6 +105,10 @@ MEDIA_CACHE = MediaCache(GRIDFS, USER_AGENT) def create_indexes(): DB.activities.create_index([("remote_id", pymongo.ASCENDING)]) DB.activities.create_index([("activity.object.id", pymongo.ASCENDING)]) + DB.activities.create_index([ + ("activity.object.id", pymongo.ASCENDING), + ("meta.deleted", pymongo.ASCENDING), + ]) # Index for the block query DB.activities.create_index( diff --git a/sass/base_theme.scss b/sass/base_theme.scss index cdc7665..17fa02c 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -324,3 +324,6 @@ input[type=submit] { color: $primary-color; text-transform: uppercase; } +.note-video { + margin: 30px 0 10px 0; +} diff --git a/templates/lookup.html b/templates/lookup.html index 31932c1..8a4173d 100644 --- a/templates/lookup.html +++ b/templates/lookup.html @@ -29,7 +29,7 @@ {{ utils.display_actor_inline(data, size=80) }} {% elif data | has_type('Create') %} {{ utils.display_note(data.object, ui=True) }} - {% elif data | has_type('Note') %} + {% elif data | has_type(['Note', 'Article', 'Video']) %} {{ utils.display_note(data, ui=True) }} {% elif data | has_type('Announce') %} {% set boost_actor = meta.actor %} diff --git a/templates/utils.html b/templates/utils.html index 6dc11e1..e54e394 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -36,7 +36,7 @@
          {{ actor.name or actor.preferredUsername }} - @{% if not no_color and obj.id | is_from_outbox %}{{ actor.preferredUsername }}{% else %}{{ actor.preferredUsername }}{% endif %}@{% if not no_color and obj.id | is_from_outbox %}{{ actor | url_or_id | get_url | domain }}{% else %}{{ actor | url_or_id | get_url | domain }}{% endif %} +
        • @{% if not no_color and obj.id | is_from_outbox %}{{ actor.preferredUsername }}{% else %}{{ actor.preferredUsername }}{% endif %}@{% if not no_color and obj.id | is_from_outbox %}{{ actor | url_or_id | get_url | domain }}{% else %}{{ actor | url_or_id | get_url | domain }}{% endif %} {% if not perma %} @@ -46,7 +46,14 @@ {% endif %}
          • {% if obj.summary %}

          {{ obj.summary | clean }}

          {% endif %} + {% if obj | has_type('Video') %} +
          + +
          + {% endif %}
          + {% if obj | has_type('Article') %} {{ obj.name }} {{ obj | url_or_id | get_url }} {% else %} @@ -57,12 +64,14 @@ {% if obj.attachment and obj | has_type('Note') %}
          {% if obj.attachment | not_only_imgs %} -

          Attachment

          -
            +

            Attachments

            +
              {% endif %} {% for a in obj.attachment %} {% if (a.mediaType and a.mediaType.startswith("image/")) or (a.type and a.type == 'Image') %} + {% elif (a.mediaType and a.mediaType.startswith("video/")) %} +
              • {% else %}
            • {% if a.filename %}{{ a.filename }}{% else %}{{ a.url }}{% endif %}
              • {% endif %} From 408308f97f37d6180b1a10bf40e71d0848f06c2f Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 2 Sep 2018 19:51:40 +0200 Subject: [PATCH 0318/1425] Fix template --- templates/utils.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/utils.html b/templates/utils.html index e54e394..617d86d 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -36,7 +36,7 @@
              {{ actor.name or actor.preferredUsername }} -
            • @{% if not no_color and obj.id | is_from_outbox %}{{ actor.preferredUsername }}{% else %}{{ actor.preferredUsername }}{% endif %}@{% if not no_color and obj.id | is_from_outbox %}{{ actor | url_or_id | get_url | domain }}{% else %}{{ actor | url_or_id | get_url | domain }}{% endif %} + @{% if not no_color and obj.id | is_from_outbox %}{{ actor.preferredUsername }}{% else %}{{ actor.preferredUsername }}{% endif %}@{% if not no_color and obj.id | is_from_outbox %}{{ actor | url_or_id | get_url | domain }}{% else %}{{ actor | url_or_id | get_url | domain }}{% endif %} {% if not perma %} From f100750382d8c3e34c55ef2e78e9bd60acbdbbd0 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 2 Sep 2018 20:32:15 +0200 Subject: [PATCH 0319/1425] Start delete support for actors --- app.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app.py b/app.py index 212702e..130a328 100644 --- a/app.py +++ b/app.py @@ -1637,6 +1637,16 @@ def inbox(): ) try: data = get_backend().fetch_iri(data["id"]) + except ActivityGoneError: + # XXX Mastodon sends Delete activities that are not dereferencable, it's the actor url with #delete + # appended, so an `ActivityGoneError` kind of ensure it's "legit" + if data["type"] == ActivityType.DELETE.value and data["id"].startswith(data["object"]): + logger.info(f"received a Delete for an actor {data!r}") + if get_backend().inbox_check_duplicate(MY_PERSON, data["id"]): + # The activity is already in the inbox + logger.info(f"received duplicate activity {data!r}, dropping it") + + get_backend().save(Box.INBOX, data) except Exception: logger.exception(f'failed to fetch remote id at {data["id"]}') return Response( From 8e43f4a23077bcf6b0bdc0fc76d243a997c2d61a Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 3 Sep 2018 08:20:43 +0200 Subject: [PATCH 0320/1425] Fix the Delete support for actors --- app.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/app.py b/app.py index 130a328..7c2883b 100644 --- a/app.py +++ b/app.py @@ -289,10 +289,10 @@ def url_or_id(d): @app.template_filter() def get_url(u): - print(f'GET_URL({u!r})') + print(f"GET_URL({u!r})") if isinstance(u, list): for l in u: - if l.get('mimeType') == 'text/html': + if l.get("mimeType") == "text/html": u = l if isinstance(u, dict): return u["href"] @@ -1640,13 +1640,25 @@ def inbox(): except ActivityGoneError: # XXX Mastodon sends Delete activities that are not dereferencable, it's the actor url with #delete # appended, so an `ActivityGoneError` kind of ensure it's "legit" - if data["type"] == ActivityType.DELETE.value and data["id"].startswith(data["object"]): + if data["type"] == ActivityType.DELETE.value and data["id"].startswith( + data["object"] + ): logger.info(f"received a Delete for an actor {data!r}") if get_backend().inbox_check_duplicate(MY_PERSON, data["id"]): # The activity is already in the inbox logger.info(f"received duplicate activity {data!r}, dropping it") - get_backend().save(Box.INBOX, data) + DB.activities.insert_one( + { + "box": Box.INBOX.value, + "activity": data, + "type": _to_list(data["type"]), + "remote_id": data["id"], + "meta": {"undo": False, "deleted": False}, + } + ) + # TODO(tsileo): write the callback the the delete external actor event + return Response(status=201) except Exception: logger.exception(f'failed to fetch remote id at {data["id"]}') return Response( From 2ee3fc0c670e032ac2ad1c5b8ad3044c98872eba Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 3 Sep 2018 09:38:29 +0200 Subject: [PATCH 0321/1425] Start playing with caching --- app.py | 15 ++++++++++++++- config.py | 2 ++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 7c2883b..1da742d 100644 --- a/app.py +++ b/app.py @@ -820,6 +820,12 @@ def paginated_query(db, q, limit=25, sort_key="_id"): def index(): if is_api_request(): return jsonify(**ME) + logged_in = session.get("logged_in", False) + if not logged_in: + cached = DB.cache.find_one({"path": request.path, "type": "html"}) + if cached: + app.logger.info("from cache") + return cached['response_data'] q = { "box": Box.OUTBOX.value, @@ -846,13 +852,20 @@ def index(): DB.activities, q, limit=25 - len(pinned) ) - return render_template( + resp = render_template( "index.html", outbox_data=outbox_data, older_than=older_than, newer_than=newer_than, pinned=pinned, ) + if not logged_in: + DB.cache.update_one( + {"path": request.path, "type": "html"}, + {"$set": {"response_data": resp, "date": datetime.now(timezone.utc)}}, + upsert=True, + ) + return resp @app.route("/with_replies") diff --git a/config.py b/config.py index a5940ce..dfb553c 100644 --- a/config.py +++ b/config.py @@ -109,6 +109,8 @@ def create_indexes(): ("activity.object.id", pymongo.ASCENDING), ("meta.deleted", pymongo.ASCENDING), ]) + DB.cache.create_index([("path", pymongo.ASCENDING), ("type", pymongo.ASCENDING)]) + DB.cache.create_index("date", expireAfterSeconds=60) # Index for the block query DB.activities.create_index( From 7237fbcc68358efa8633628c3d2af81215fce943 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 3 Sep 2018 20:21:33 +0200 Subject: [PATCH 0322/1425] Improve caching --- app.py | 97 +++++++++++++++++++++++++++--------------- config.py | 4 +- docker-compose-dev.yml | 11 +++++ requirements.txt | 1 + tasks.py | 30 ++++++++++++- 5 files changed, 106 insertions(+), 37 deletions(-) diff --git a/app.py b/app.py index 1da742d..30c8241 100644 --- a/app.py +++ b/app.py @@ -816,16 +816,41 @@ def paginated_query(db, q, limit=25, sort_key="_id"): return outbox_data, older_than, newer_than +CACHING = True + + +def _get_cached(type_="html", arg=None): + if not CACHING: + return None + logged_in = session.get("logged_in") + if not logged_in: + cached = DB.cache2.find_one({"path": request.path, "type": type_, "arg": arg}) + if cached: + app.logger.info("from cache") + return cached['response_data'] + return None + +def _cache(resp, type_="html", arg=None): + if not CACHING: + return None + logged_in = session.get("logged_in") + if not logged_in: + DB.cache2.update_one( + {"path": request.path, "type": type_, "arg": arg}, + {"$set": {"response_data": resp, "date": datetime.now(timezone.utc)}}, + upsert=True, + ) + return None + + @app.route("/") def index(): if is_api_request(): return jsonify(**ME) - logged_in = session.get("logged_in", False) - if not logged_in: - cached = DB.cache.find_one({"path": request.path, "type": "html"}) - if cached: - app.logger.info("from cache") - return cached['response_data'] + cache_arg = f"{request.args.get('older_than', '')}:{request.args.get('newer_than', '')}" + cached = _get_cached("html", cache_arg) + if cached: + return cached q = { "box": Box.OUTBOX.value, @@ -859,12 +884,7 @@ def index(): newer_than=newer_than, pinned=pinned, ) - if not logged_in: - DB.cache.update_one( - {"path": request.path, "type": "html"}, - {"$set": {"response_data": resp, "date": datetime.now(timezone.utc)}}, - upsert=True, - ) + _cache(resp, "html", cache_arg) return resp @@ -1011,32 +1031,41 @@ def note_by_id(note_id): @app.route("/nodeinfo") def nodeinfo(): - q = { - "box": Box.OUTBOX.value, - "meta.deleted": False, # TODO(tsileo): retrieve deleted and expose tombstone - "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, - } + response = _get_cached("api") + cached = True + if not response: + cached = False + q = { + "box": Box.OUTBOX.value, + "meta.deleted": False, # TODO(tsileo): retrieve deleted and expose tombstone + "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, + } + + response = json.dumps( + { + "version": "2.0", + "software": { + "name": "microblogpub", + "version": f"Microblog.pub {VERSION}", + }, + "protocols": ["activitypub"], + "services": {"inbound": [], "outbound": []}, + "openRegistrations": False, + "usage": {"users": {"total": 1}, "localPosts": DB.activities.count(q)}, + "metadata": { + "sourceCode": "https://github.com/tsileo/microblog.pub", + "nodeName": f"@{USERNAME}@{DOMAIN}", + }, + } + ) + + if not cached: + _cache(response, "api") return Response( headers={ "Content-Type": "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#" }, - response=json.dumps( - { - "version": "2.0", - "software": { - "name": "microblogpub", - "version": f"Microblog.pub {VERSION}", - }, - "protocols": ["activitypub"], - "services": {"inbound": [], "outbound": []}, - "openRegistrations": False, - "usage": {"users": {"total": 1}, "localPosts": DB.activities.count(q)}, - "metadata": { - "sourceCode": "https://github.com/tsileo/microblog.pub", - "nodeName": f"@{USERNAME}@{DOMAIN}", - }, - } - ), + response=response, ) diff --git a/config.py b/config.py index dfb553c..b66ce71 100644 --- a/config.py +++ b/config.py @@ -109,8 +109,8 @@ def create_indexes(): ("activity.object.id", pymongo.ASCENDING), ("meta.deleted", pymongo.ASCENDING), ]) - DB.cache.create_index([("path", pymongo.ASCENDING), ("type", pymongo.ASCENDING)]) - DB.cache.create_index("date", expireAfterSeconds=60) + DB.cache2.create_index([("path", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ("arg", pymongo.ASCENDING)]) + DB.cache2.create_index("date", expireAfterSeconds=3600*12) # Index for the block query DB.activities.create_index( diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index b487dc9..8a3a677 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -1,5 +1,16 @@ version: '3' services: + flower: + image: microblogpub:latest + links: + - mongo + - rabbitmq + command: 'celery flower -l info -A tasks --broker amqp://guest@rabbitmq// --address=0.0.0.0 --port=5556' + environment: + - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rabbitmq// + - MICROBLOGPUB_MONGODB_HOST=mongo:27017 + ports: + - "5556:5556" celery: image: microblogpub:latest links: diff --git a/requirements.txt b/requirements.txt index d2eec2d..d9fccea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ python-dateutil libsass +flower gunicorn piexif requests diff --git a/tasks.py b/tasks.py index c05d200..96b185b 100644 --- a/tasks.py +++ b/tasks.py @@ -6,6 +6,7 @@ import random import requests from celery import Celery from little_boxes import activitypub as ap +from little_boxes.errors import BadActivityError from little_boxes.errors import ActivityGoneError from little_boxes.errors import ActivityNotFoundError from little_boxes.errors import NotAnActivityError @@ -59,7 +60,8 @@ def process_new_activity(self, iri: str) -> None: try: activity.get_object() tag_stream = True - except NotAnActivityError: + except (NotAnActivityError, BadActivityError): + log.exception(f"failed to get announce object for {activity!r}") # Most likely on OStatus notice tag_stream = False should_delete = True @@ -317,6 +319,26 @@ def post_to_inbox(activity: ap.BaseActivity) -> None: finish_post_to_inbox.delay(activity.id) +def invalidate_cache(activity): + if activity.has_type(ap.ActivityType.LIKE): + if activity.get_object().id.startswith(BASE_URL): + DB.cache2.remove() + elif activity.has_type(ap.ActivityType.ANNOUNCE): + if activity.get_object.id.startswith(BASE_URL): + DB.cache2.remove() + elif activity.has_type(ap.ActivityType.UNDO): + DB.cache2.remove() + elif activity.has_type(ap.ActivityType.DELETE): + # TODO(tsileo): only invalidate if it's a delete of a reply + DB.cache2.remove() + elif activity.has_type(ap.ActivityType.UPDATE): + DB.cache2.remove() + elif activity.has_type(ap.ActivityType.CREATE): + note = activity.get_object() + if not note.inReplyTo or note.inReplyTo.startswith(ID): + DB.cache2.remove() + # FIXME(tsileo): check if it's a reply of a reply + @app.task(bind=True, max_retries=MAX_RETRIES) # noqa: C901 def finish_post_to_inbox(self, iri: str) -> None: try: @@ -345,6 +367,10 @@ def finish_post_to_inbox(self, iri: str) -> None: back.inbox_undo_announce(MY_PERSON, obj) elif obj.has_type(ap.ActivityType.FOLLOW): back.undo_new_follower(MY_PERSON, obj) + try: + invalidate_cache(activity) + except Exception: + log.exception("failed to invalidate cache") except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): log.exception(f"no retry") except Exception as err: @@ -396,6 +422,8 @@ def finish_post_to_outbox(self, iri: str) -> None: log.info(f"recipients={recipients}") activity = ap.clean_activity(activity.to_dict()) + DB.cache2.remove() + payload = json.dumps(activity) for recp in recipients: log.debug(f"posting to {recp}") From e3cf7d9ee697df5665b709738f39d2f0cebae277 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 4 Sep 2018 21:25:25 +0200 Subject: [PATCH 0323/1425] Bugfix in the collection for the last empty page --- activitypub.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/activitypub.py b/activitypub.py index 6aa3be5..c7425dd 100644 --- a/activitypub.py +++ b/activitypub.py @@ -646,6 +646,16 @@ def build_ordered_collection( data = list(col.find(q, limit=limit).sort("_id", -1)) if not data: + # Returns an empty page if there's a cursor + if cursor: + return { + "@context": ap.COLLECTION_CTX, + "type": ap.ActivityType.ORDERED_COLLECTION_PAGE.value, + "id": BASE_URL + "/" + col_name + "?cursor=" + cursor, + "partOf": BASE_URL + "/" + col_name, + "totalItems": 0, + "oredredItems": [], + } return { "@context": ap.COLLECTION_CTX, "id": BASE_URL + "/" + col_name, From 7df8837b34dd7f65c4f964c2184c7c49572ee462 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 5 Sep 2018 21:47:11 +0200 Subject: [PATCH 0324/1425] Bugfix cache invalidation --- tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks.py b/tasks.py index 96b185b..7a2f64d 100644 --- a/tasks.py +++ b/tasks.py @@ -324,7 +324,7 @@ def invalidate_cache(activity): if activity.get_object().id.startswith(BASE_URL): DB.cache2.remove() elif activity.has_type(ap.ActivityType.ANNOUNCE): - if activity.get_object.id.startswith(BASE_URL): + if activity.get_object().id.startswith(BASE_URL): DB.cache2.remove() elif activity.has_type(ap.ActivityType.UNDO): DB.cache2.remove() From 665e185407cc7d5a84dc5e7ad2fbc857713a4095 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 6 Sep 2018 19:19:47 +0200 Subject: [PATCH 0325/1425] Handle reply of articles --- Makefile | 2 +- activitypub.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 6c7a767..daad832 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ reload-fed: WEB_PORT=5007 CONFIG_DIR=./tests/fixtures/instance2/config docker-compose -p instance2 -f docker-compose-tests.yml up -d --force-recreate --build reload-dev: - docker build . -t microblogpub:latest + # docker build . -t microblogpub:latest docker-compose -f docker-compose-dev.yml up -d --force-recreate update: diff --git a/activitypub.py b/activitypub.py index c7425dd..c53aaa8 100644 --- a/activitypub.py +++ b/activitypub.py @@ -463,7 +463,7 @@ class MicroblogPubBackend(Backend): new_threads = [] root_reply = in_reply_to - reply = ap.fetch_remote_activity(root_reply, expected=ap.ActivityType.NOTE) + reply = ap.fetch_remote_activity(root_reply) creply = DB.activities.find_one_and_update( {"activity.object.id": in_reply_to}, @@ -479,7 +479,7 @@ class MicroblogPubBackend(Backend): if not in_reply_to: break root_reply = in_reply_to - reply = ap.fetch_remote_activity(root_reply, expected=ap.ActivityType.NOTE) + reply = ap.fetch_remote_activity(root_reply) q = {"activity.object.id": root_reply} if not DB.activities.count(q): self.save(Box.REPLIES, reply) From 1926ba92fa03c016ff3eab21ca1dd87bd8edb7ee Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 11 Sep 2018 21:49:25 +0200 Subject: [PATCH 0326/1425] Typo in the ordered collections Fixes #39 --- activitypub.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/activitypub.py b/activitypub.py index c53aaa8..51557fa 100644 --- a/activitypub.py +++ b/activitypub.py @@ -654,14 +654,14 @@ def build_ordered_collection( "id": BASE_URL + "/" + col_name + "?cursor=" + cursor, "partOf": BASE_URL + "/" + col_name, "totalItems": 0, - "oredredItems": [], + "orderedItems": [], } return { "@context": ap.COLLECTION_CTX, "id": BASE_URL + "/" + col_name, "totalItems": 0, "type": ap.ActivityType.ORDERED_COLLECTION.value, - "oredredItems": [], + "orderedItems": [], } start_cursor = str(data[0]["_id"]) From 1a7a02a221467a8b0c4fbaa5df403cb36e70a9fc Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 24 Feb 2019 21:04:09 +0100 Subject: [PATCH 0327/1425] Bugfixes by @cwt (#42) * check for missing actor in stream item * reply on my own status * fix img exceed div * check for "actor" key in item.meta before render * force UTF-8 text encoding * check if a is a dict or string * check d object must be dict to get url or id * json/atom/rss feed * handle missing root_id * fix newer-link hover * fix older-link position * quick hack to support peertube video * enable IndieAuth endpoint * fix 500 when remote follow --- activitypub.py | 10 +++--- app.py | 73 ++++++++++++++++++++++++++++++++++--------- sass/base_theme.scss | 4 ++- templates/index.html | 2 ++ templates/stream.html | 2 ++ templates/utils.html | 2 +- utils/opengraph.py | 1 + 7 files changed, 72 insertions(+), 22 deletions(-) diff --git a/activitypub.py b/activitypub.py index 51557fa..9185745 100644 --- a/activitypub.py +++ b/activitypub.py @@ -523,8 +523,8 @@ def gen_feed(): fg.logo(ME.get("icon", {}).get("url")) fg.language("en") for item in DB.activities.find( - {"box": Box.OUTBOX.value, "type": "Create"}, limit=50 - ): + {"box": Box.OUTBOX.value, "type": "Create", "meta.deleted": False}, limit=10 + ).sort("_id", -1): fe = fg.add_entry() fe.id(item["activity"]["object"].get("url")) fe.link(href=item["activity"]["object"].get("url")) @@ -537,11 +537,11 @@ def json_feed(path: str) -> Dict[str, Any]: """JSON Feed (https://jsonfeed.org/) document.""" data = [] for item in DB.activities.find( - {"box": Box.OUTBOX.value, "type": "Create"}, limit=50 - ): + {"box": Box.OUTBOX.value, "type": "Create", "meta.deleted": False}, limit=10 + ).sort("_id", -1): data.append( { - "id": item["id"], + "id": item["activity"]["id"], "url": item["activity"]["object"].get("url"), "content_html": item["activity"]["object"]["content"], "content_text": html2text(item["activity"]["object"]["content"]), diff --git a/app.py b/app.py index 30c8241..0730813 100644 --- a/app.py +++ b/app.py @@ -189,7 +189,10 @@ ALLOWED_TAGS = [ def clean_html(html): - return bleach.clean(html, tags=ALLOWED_TAGS) + try: + return bleach.clean(html, tags=ALLOWED_TAGS) + except: + return "" _GRIDFS_CACHE: Dict[Tuple[Kind, str, Optional[int]], str] = {} @@ -282,9 +285,12 @@ def domain(url): @app.template_filter() def url_or_id(d): - if "url" in d: - return d["url"] - return d["id"] + if isinstance(d, dict): + if ("url" in d) and isinstance(d["url"], str): + return d["url"] + else: + return d["id"] + return "" @app.template_filter() @@ -367,7 +373,9 @@ def _is_img(filename): @app.template_filter() def not_only_imgs(attachment): for a in attachment: - if not _is_img(a["url"]): + if isinstance(a, dict) and not _is_img(a["url"]): + return True + if isinstance(a, str) and not _is_img(a): return True return False @@ -961,7 +969,10 @@ def _build_thread(data, include_children=True): ): _flatten(snode, level=level + 1) - _flatten(idx[root_id]) + try: + _flatten(idx[root_id]) + except KeyError: + app.logger.info(f"{root_id} is not there! skipping") return thread @@ -1524,7 +1535,15 @@ def _user_api_arg(key: str, **kwargs): def _user_api_get_note(from_outbox: bool = False): oid = _user_api_arg("id") app.logger.info(f"fetching {oid}") - note = ap.parse_activity(get_backend().fetch_iri(oid), expected=ActivityType.NOTE) + try: + note = ap.parse_activity(get_backend().fetch_iri(oid), expected=ActivityType.NOTE) + except: + try: + note = ap.parse_activity(get_backend().fetch_iri(oid), expected=ActivityType.VIDEO) + except: + raise ActivityNotFoundError( + "Expected Note or Video ActivityType, but got something else" + ) if from_outbox and not note.id.startswith(ID): raise NotFromOutboxError( f"cannot load {note.id}, id must be owned by the server" @@ -1876,12 +1895,8 @@ def followers(): ) raw_followers, older_than, newer_than = paginated_query(DB.activities, q) - followers = [] - for doc in raw_followers: - try: - followers.append(doc["meta"]["actor"]) - except Exception: - pass + followers = [doc["meta"]["actor"] + for doc in raw_followers if "actor" in doc.get("meta", {})] return render_template( "followers.html", followers_data=followers, @@ -1909,7 +1924,9 @@ def following(): abort(404) following, older_than, newer_than = paginated_query(DB.activities, q) - following = [(doc["remote_id"], doc["meta"]["object"]) for doc in following] + following = [(doc["remote_id"], doc["meta"]["object"]) + for doc in following + if "remote_id" in doc and "object" in doc.get("meta", {})] return render_template( "following.html", following_data=following, @@ -2070,7 +2087,7 @@ def indieauth_flow(): return redirect(red) -# @app.route('/indieauth', methods=['GET', 'POST']) +@app.route('/indieauth', methods=['GET', 'POST']) def indieauth_endpoint(): if request.method == "GET": if not session.get("logged_in"): @@ -2167,3 +2184,29 @@ def token_endpoint(): "client_id": payload["client_id"], } ) + + +@app.route("/feed.json") +def json_feed(): + return Response( + response=json.dumps( + activitypub.json_feed("/feed.json") + ), + headers={"Content-Type": "application/json"}, + ) + + +@app.route("/feed.atom") +def atom_feed(): + return Response( + response=activitypub.gen_feed().atom_str(), + headers={"Content-Type": "application/atom+xml"}, + ) + + +@app.route("/feed.rss") +def rss_feed(): + return Response( + response=activitypub.gen_feed().rss_str(), + headers={"Content-Type": "application/rss+xml"}, + ) diff --git a/sass/base_theme.scss b/sass/base_theme.scss index 17fa02c..b878a19 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -40,10 +40,11 @@ a:hover { .lcolor { color: $color-light; } -.older-link, .newer-linker, .older-link:hover, .newer-link:hover { +.older-link, .newer-link, .older-link:hover, .newer-link:hover { text-decoration: none; padding: 3px; } +.older-link { float: left } .newer-link { float: right } .clear { clear: both; } .remote-follow-button { @@ -210,6 +211,7 @@ a:hover { .note-wrapper { flex: 1; padding-left: 15px; + overflow: hidden; } .bottom-bar { margin-top:10px; } diff --git a/templates/index.html b/templates/index.html index 1331a1f..13a488c 100644 --- a/templates/index.html +++ b/templates/index.html @@ -34,6 +34,7 @@ {% for item in outbox_data %} {% if item | has_type('Announce') %} + {% if "actor" in item.meta %} {% set boost_actor = item.meta.actor %} {% if session.logged_in %}
              @@ -50,6 +51,7 @@ {{ boost_actor.name }} boosted

              {% endif %} + {% endif %} {% if item.meta.object %} {{ utils.display_note(item.meta.object, ui=False, meta={'actor': item.meta.object_actor}) }} {% endif %} diff --git a/templates/stream.html b/templates/stream.html index 37d1394..3aa5ea5 100644 --- a/templates/stream.html +++ b/templates/stream.html @@ -8,6 +8,7 @@
              {% for item in inbox_data %} + {% if 'actor' in item.meta %} {% if item | has_type('Create') %} {{ utils.display_note(item.activity.object, ui=True, meta=item.meta) }} {% else %} @@ -53,6 +54,7 @@ {% endif %} + {% endif %} {% endif %} {% endfor %} diff --git a/templates/utils.html b/templates/utils.html index 617d86d..56eab23 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -129,7 +129,6 @@ {% if session.logged_in %} {% if ui%} -reply {% if meta.boosted %} @@ -195,6 +194,7 @@ {% endif %} +reply {% endif %} diff --git a/utils/opengraph.py b/utils/opengraph.py index 318b1bc..df59732 100644 --- a/utils/opengraph.py +++ b/utils/opengraph.py @@ -48,6 +48,7 @@ def fetch_og_metadata(user_agent, links): logger.debug(f"skipping {l}") continue + r.encoding = 'UTF-8' html = r.text try: data = dict(opengraph.OpenGraph(html=html)) From 5b38c5e723d45339669317599f1ebd988ca4f3d0 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 24 Feb 2019 21:34:09 +0100 Subject: [PATCH 0328/1425] Fix CI --- .travis.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5094276..2e8ab86 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,12 @@ language: python sudo: required python: - - "3.6" + - '3.7' +matrix: + include: + - python: 3.7 + dist: xenial + sudo: true services: - docker install: @@ -23,8 +28,6 @@ script: - WEB_PORT=5007 CONFIG_DIR=./tests/fixtures/instance2/config docker-compose -p instance2 -f docker-compose-tests.yml up -d - docker-compose -p instance2 -f docker-compose-tests.yml ps - sleep 5 - - curl http://localhost:5006 - - curl http://localhost:5007 # Integration tests first - python -m pytest -v --ignore data -k integration # Federation tests (with two local instances) From 552344a2c5591d60498a47169d10363e5cab626b Mon Sep 17 00:00:00 2001 From: Chaiwat Suttipongsakul Date: Tue, 26 Feb 2019 16:53:14 +0700 Subject: [PATCH 0329/1425] Stay on the same page after pin,link,boost,delete,block (#43) --- templates/utils.html | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/templates/utils.html b/templates/utils.html index 56eab23..3f54f3d 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -115,7 +115,15 @@ {% if session.logged_in %} {% set perma_id = obj.id | permalink_id %} + +{% if request.args.get('older_than') %} +{% set redir = request.path + "?older_than=" + request.args.get('older_than') + "#activity-" + perma_id %} +{% elif request.args.get('newer_than') %} +{% set redir = request.path + "?newer_than=" + request.args.get('newer_than') + "#activity-" + perma_id %} +{% else %} {% set redir = request.path + "#activity-" + perma_id %} +{% endif %} + {% set aid = obj.id | quote_plus %} {% endif %} From 01b849be70d56b61b0bc2cdff52db4f7b828e5e4 Mon Sep 17 00:00:00 2001 From: Chaiwat Suttipongsakul Date: Sun, 3 Mar 2019 05:31:16 +0700 Subject: [PATCH 0330/1425] set tornado to version older than 6.0.0 (#44) --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index d9fccea..3645578 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ python-dateutil libsass +tornado<6.0.0 flower gunicorn piexif From 066309a0c820dfbbde7a83282bbbb197dd32837d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 11:35:48 +0200 Subject: [PATCH 0331/1425] Try poussetaches --- .travis.yml | 1 + app.py | 646 ++++++++++++++++++++++++++++++++++++--- config.py | 17 +- docker-compose-tests.yml | 11 +- docker-compose.yml | 6 + poussetaches.py | 48 +++ tasks.py | 1 + 7 files changed, 673 insertions(+), 57 deletions(-) create mode 100644 poussetaches.py diff --git a/.travis.yml b/.travis.yml index 2e8ab86..db184e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ install: - sudo chmod +x /usr/local/bin/docker-compose - docker-compose --version - pip install -r dev-requirements.txt + - git clone https://github.com/tsileo/poussetaches.git && cd poussetaches && docker build . -t poussetaches:latest && cd - script: - mypy --ignore-missing-imports . - flake8 activitypub.py diff --git a/app.py b/app.py index 0730813..c9ac063 100644 --- a/app.py +++ b/app.py @@ -16,6 +16,8 @@ from typing import Tuple from urllib.parse import urlencode from urllib.parse import urlparse +from requests.exceptions import HTTPError +import requests import bleach import mf2py import pymongo @@ -41,7 +43,10 @@ from little_boxes.activitypub import _to_list from little_boxes.activitypub import clean_activity from little_boxes.activitypub import get_backend from little_boxes.content_helper import parse_markdown +from little_boxes.linked_data_sig import generate_signature from little_boxes.errors import ActivityGoneError +from little_boxes.errors import NotAnActivityError +from little_boxes.errors import BadActivityError from little_boxes.errors import ActivityNotFoundError from little_boxes.errors import Error from little_boxes.errors import NotFromOutboxError @@ -49,15 +54,18 @@ from little_boxes.httpsig import HTTPSigAuth from little_boxes.httpsig import verify_request from little_boxes.webfinger import get_actor_url from little_boxes.webfinger import get_remote_follow_template +from utils import opengraph from passlib.hash import bcrypt from u2flib_server import u2f from werkzeug.utils import secure_filename import activitypub import config -import tasks + +# import tasks from activitypub import Box from activitypub import embed_collection +from config import USER_AGENT from config import ADMIN_API_KEY from config import BASE_URL from config import DB @@ -78,6 +86,11 @@ from utils.key import get_secret_key from utils.lookup import lookup from utils.media import Kind +from poussetaches import PousseTaches + +p = PousseTaches("http://poussetaches:7991", "http://web:5005") + + back = activitypub.MicroblogPubBackend() ap.use_backend(back) @@ -191,7 +204,7 @@ ALLOWED_TAGS = [ def clean_html(html): try: return bleach.clean(html, tags=ALLOWED_TAGS) - except: + except Exception: return "" @@ -631,7 +644,7 @@ def authorize_follow(): return redirect("/following") follow = ap.Follow(actor=MY_PERSON.id, object=actor) - tasks.post_to_outbox(follow) + post_to_outbox(follow) return redirect("/following") @@ -758,7 +771,7 @@ def tmp_migrate4(): @login_required def tmp_migrate5(): for activity in DB.activities.find(): - tasks.cache_actor.delay(activity["remote_id"], also_cache_attachments=False) + Tasks.cache_actor(activity["remote_id"], also_cache_attachments=False) return "Done" @@ -835,9 +848,10 @@ def _get_cached(type_="html", arg=None): cached = DB.cache2.find_one({"path": request.path, "type": type_, "arg": arg}) if cached: app.logger.info("from cache") - return cached['response_data'] + return cached["response_data"] return None + def _cache(resp, type_="html", arg=None): if not CACHING: return None @@ -855,7 +869,9 @@ def _cache(resp, type_="html", arg=None): def index(): if is_api_request(): return jsonify(**ME) - cache_arg = f"{request.args.get('older_than', '')}:{request.args.get('newer_than', '')}" + cache_arg = ( + f"{request.args.get('older_than', '')}:{request.args.get('newer_than', '')}" + ) cached = _get_cached("html", cache_arg) if cached: return cached @@ -1053,22 +1069,22 @@ def nodeinfo(): } response = json.dumps( - { - "version": "2.0", - "software": { - "name": "microblogpub", - "version": f"Microblog.pub {VERSION}", - }, - "protocols": ["activitypub"], - "services": {"inbound": [], "outbound": []}, - "openRegistrations": False, - "usage": {"users": {"total": 1}, "localPosts": DB.activities.count(q)}, - "metadata": { - "sourceCode": "https://github.com/tsileo/microblog.pub", - "nodeName": f"@{USERNAME}@{DOMAIN}", - }, - } - ) + { + "version": "2.0", + "software": { + "name": "microblogpub", + "version": f"Microblog.pub {VERSION}", + }, + "protocols": ["activitypub"], + "services": {"inbound": [], "outbound": []}, + "openRegistrations": False, + "usage": {"users": {"total": 1}, "localPosts": DB.activities.count(q)}, + "metadata": { + "sourceCode": "https://github.com/tsileo/microblog.pub", + "nodeName": f"@{USERNAME}@{DOMAIN}", + }, + } + ) if not cached: _cache(response, "api") @@ -1197,7 +1213,7 @@ def outbox(): data = request.get_json(force=True) print(data) activity = ap.parse_activity(data) - activity_id = tasks.post_to_outbox(activity) + activity_id = post_to_outbox(activity) return Response(status=201, headers={"Location": activity_id}) @@ -1536,11 +1552,15 @@ def _user_api_get_note(from_outbox: bool = False): oid = _user_api_arg("id") app.logger.info(f"fetching {oid}") try: - note = ap.parse_activity(get_backend().fetch_iri(oid), expected=ActivityType.NOTE) - except: + note = ap.parse_activity( + get_backend().fetch_iri(oid), expected=ActivityType.NOTE + ) + except Exception: try: - note = ap.parse_activity(get_backend().fetch_iri(oid), expected=ActivityType.VIDEO) - except: + note = ap.parse_activity( + get_backend().fetch_iri(oid), expected=ActivityType.VIDEO + ) + except Exception: raise ActivityNotFoundError( "Expected Note or Video ActivityType, but got something else" ) @@ -1570,7 +1590,7 @@ def api_delete(): delete = ap.Delete(actor=ID, object=ap.Tombstone(id=note.id).to_dict(embed=True)) - delete_id = tasks.post_to_outbox(delete) + delete_id = post_to_outbox(delete) return _user_api_response(activity=delete_id) @@ -1581,7 +1601,7 @@ def api_boost(): note = _user_api_get_note() announce = note.build_announce(MY_PERSON) - announce_id = tasks.post_to_outbox(announce) + announce_id = post_to_outbox(announce) return _user_api_response(activity=announce_id) @@ -1592,7 +1612,7 @@ def api_like(): note = _user_api_get_note() like = note.build_like(MY_PERSON) - like_id = tasks.post_to_outbox(like) + like_id = post_to_outbox(like) return _user_api_response(activity=like_id) @@ -1639,7 +1659,7 @@ def api_undo(): obj = ap.parse_activity(doc.get("activity")) # FIXME(tsileo): detect already undo-ed and make this API call idempotent undo = obj.build_undo() - undo_id = tasks.post_to_outbox(undo) + undo_id = post_to_outbox(undo) return _user_api_response(activity=undo_id) @@ -1664,7 +1684,7 @@ def admin_stream(): ) -@app.route("/inbox", methods=["GET", "POST"]) +@app.route("/inbox", methods=["GET", "POST"]) # noqa: C901 def inbox(): if request.method == "GET": if not is_api_request(): @@ -1733,7 +1753,7 @@ def inbox(): ) activity = ap.parse_activity(data) logger.debug(f"inbox activity={activity}/{data}") - tasks.post_to_inbox(activity) + post_to_inbox(activity) return Response(status=201) @@ -1819,7 +1839,7 @@ def api_new_note(): note = ap.Note(**raw_note) create = note.build_create() - create_id = tasks.post_to_outbox(create) + create_id = post_to_outbox(create) return _user_api_response(activity=create_id) @@ -1852,7 +1872,7 @@ def api_block(): return _user_api_response(activity=existing["activity"]["id"]) block = ap.Block(actor=MY_PERSON.id, object=actor) - block_id = tasks.post_to_outbox(block) + block_id = post_to_outbox(block) return _user_api_response(activity=block_id) @@ -1874,7 +1894,7 @@ def api_follow(): return _user_api_response(activity=existing["activity"]["id"]) follow = ap.Follow(actor=MY_PERSON.id, object=actor) - follow_id = tasks.post_to_outbox(follow) + follow_id = post_to_outbox(follow) return _user_api_response(activity=follow_id) @@ -1895,8 +1915,9 @@ def followers(): ) raw_followers, older_than, newer_than = paginated_query(DB.activities, q) - followers = [doc["meta"]["actor"] - for doc in raw_followers if "actor" in doc.get("meta", {})] + followers = [ + doc["meta"]["actor"] for doc in raw_followers if "actor" in doc.get("meta", {}) + ] return render_template( "followers.html", followers_data=followers, @@ -1924,9 +1945,11 @@ def following(): abort(404) following, older_than, newer_than = paginated_query(DB.activities, q) - following = [(doc["remote_id"], doc["meta"]["object"]) - for doc in following - if "remote_id" in doc and "object" in doc.get("meta", {})] + following = [ + (doc["remote_id"], doc["meta"]["object"]) + for doc in following + if "remote_id" in doc and "object" in doc.get("meta", {}) + ] return render_template( "following.html", following_data=following, @@ -2087,7 +2110,7 @@ def indieauth_flow(): return redirect(red) -@app.route('/indieauth', methods=['GET', 'POST']) +@app.route("/indieauth", methods=["GET", "POST"]) def indieauth_endpoint(): if request.method == "GET": if not session.get("logged_in"): @@ -2189,9 +2212,7 @@ def token_endpoint(): @app.route("/feed.json") def json_feed(): return Response( - response=json.dumps( - activitypub.json_feed("/feed.json") - ), + response=json.dumps(activitypub.json_feed("/feed.json")), headers={"Content-Type": "application/json"}, ) @@ -2210,3 +2231,538 @@ def rss_feed(): response=activitypub.gen_feed().rss_str(), headers={"Content-Type": "application/rss+xml"}, ) + + +@app.route("/task/t1") +def task_t1(): + p.push( + "https://mastodon.cloud/@iulius/101852467780804071/activity", + "/task/cache_object", + ) + return "ok" + + +@app.route("/task/t2", methods=["POST"]) +def task_t2(): + print(request) + print(request.headers) + task = p.parse(request) + print(task) + return "yay" + + +@app.route("/task/fetch_og_meta", methods=["POST"]) +def task_fetch_og_metadata(): + task = p.parse(request) + app.logger.info(f"task={task!r}") + iri = task.payload + try: + activity = ap.fetch_remote_activity(iri) + app.logger.info(f"activity={activity!r}") + if activity.has_type(ap.ActivityType.CREATE): + note = activity.get_object() + links = opengraph.links_from_note(note.to_dict()) + og_metadata = opengraph.fetch_og_metadata(USER_AGENT, links) + for og in og_metadata: + if not og.get("image"): + continue + MEDIA_CACHE.cache_og_image(og["image"]) + + app.logger.debug(f"OG metadata {og_metadata!r}") + DB.activities.update_one( + {"remote_id": iri}, {"$set": {"meta.og_metadata": og_metadata}} + ) + + app.logger.info(f"OG metadata fetched for {iri}") + except (ActivityGoneError, ActivityNotFoundError): + app.logger.exception(f"dropping activity {iri}, skip OG metedata") + return "" + except requests.exceptions.HTTPError as http_err: + if 400 <= http_err.response.status_code < 500: + app.logger.exception("bad request, no retry") + return "" + app.logger.exception("failed to fetch OG metadata") + abort(500) + except Exception: + app.logger.exception(f"failed to fetch OG metadata for {iri}") + abort(500) + + +@app.route("/task/cache_object", methods=["POST"]) +def task_cache_object(): + task = p.parse(request) + app.logger.info(f"task={task!r}") + iri = task.payload + try: + activity = ap.fetch_remote_activity(iri) + print(activity) + print(activity.__dict__) + app.logger.info(f"activity={activity!r}") + obj = activity + # obj = activity.get_object() + DB.activities.update_one( + {"remote_id": activity.id}, + { + "$set": { + "meta.object": obj.to_dict(embed=True), + "meta.object_actor": activitypub._actor_to_meta(obj.get_actor()), + } + }, + ) + except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): + DB.activities.update_one({"remote_id": iri}, {"$set": {"meta.deleted": True}}) + app.logger.exception(f"flagging activity {iri} as deleted, no object caching") + return "" + except Exception: + app.logger.exception(f"failed to cache object for {iri}") + abort(500) + return "" + + +class Tasks: + @staticmethod + def cache_object(iri: str) -> None: + p.push(iri, "/task/cache_object") + + @staticmethod + def cache_actor(iri: str, also_cache_attachments: bool = True) -> None: + p.push( + {"iri": iri, "also_cache_attachments": also_cache_attachments}, + "/task/cache_actor", + ) + + @staticmethod + def post_to_remote_inbox(payload: str, recp: str) -> None: + p.push({"payload": payload, "to": recp}, "/task/post_to_remote_inbox") + + @staticmethod + def forward_activity(iri: str) -> None: + p.push(iri, "/task/forward_activity") + + @staticmethod + def fetch_og_meta(iri: str) -> None: + p.push(iri, "/task/fetch_og_meta") + + @staticmethod + def process_new_activity(iri: str) -> None: + p.push(iri, "/task/process_new_activity") + + @staticmethod + def cache_attachments(iri: str) -> None: + p.push(iri, "/task/cache_attachments") + + @staticmethod + def finish_post_to_inbox(iri: str) -> None: + p.push(iri, "/task/finish_post_to_inbox") + + @staticmethod + def finish_post_to_outbox(iri: str) -> None: + p.push(iri, "/task/finish_post_to_outbox") + + +@app.route("/task/finish_post_to_outbox", methods=["POST"]) # noqa:C901 +def task_finish_post_to_outbox(): + task = p.parse(request) + app.logger.info(f"task={task!r}") + iri = task.payload + try: + activity = ap.fetch_remote_activity(iri) + app.logger.info(f"activity={activity!r}") + + recipients = activity.recipients() + + if activity.has_type(ap.ActivityType.DELETE): + back.outbox_delete(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.UPDATE): + back.outbox_update(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.CREATE): + back.outbox_create(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.ANNOUNCE): + back.outbox_announce(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.LIKE): + back.outbox_like(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.UNDO): + obj = activity.get_object() + if obj.has_type(ap.ActivityType.LIKE): + back.outbox_undo_like(MY_PERSON, obj) + elif obj.has_type(ap.ActivityType.ANNOUNCE): + back.outbox_undo_announce(MY_PERSON, obj) + elif obj.has_type(ap.ActivityType.FOLLOW): + back.undo_new_following(MY_PERSON, obj) + + app.logger.info(f"recipients={recipients}") + activity = ap.clean_activity(activity.to_dict()) + + DB.cache2.remove() + + payload = json.dumps(activity) + for recp in recipients: + app.logger.debug(f"posting to {recp}") + Tasks.post_to_remote_inbox(payload, recp) + except (ActivityGoneError, ActivityNotFoundError): + app.logger.exception(f"no retry") + except Exception: + app.logger.exception(f"failed to post to remote inbox for {iri}") + abort(500) + + +@app.route("/task/finish_post_to_inbox", methods=["POST"]) # noqa: C901 +def task_finish_post_to_inbox(): + task = p.parse(request) + app.logger.info(f"task={task!r}") + iri = task.payload + try: + activity = ap.fetch_remote_activity(iri) + app.logger.info(f"activity={activity!r}") + + if activity.has_type(ap.ActivityType.DELETE): + back.inbox_delete(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.UPDATE): + back.inbox_update(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.CREATE): + back.inbox_create(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.ANNOUNCE): + back.inbox_announce(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.LIKE): + back.inbox_like(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.FOLLOW): + # Reply to a Follow with an Accept + accept = ap.Accept(actor=ID, object=activity.to_dict(embed=True)) + post_to_outbox(accept) + elif activity.has_type(ap.ActivityType.UNDO): + obj = activity.get_object() + if obj.has_type(ap.ActivityType.LIKE): + back.inbox_undo_like(MY_PERSON, obj) + elif obj.has_type(ap.ActivityType.ANNOUNCE): + back.inbox_undo_announce(MY_PERSON, obj) + elif obj.has_type(ap.ActivityType.FOLLOW): + back.undo_new_follower(MY_PERSON, obj) + try: + invalidate_cache(activity) + except Exception: + app.logger.exception("failed to invalidate cache") + except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): + app.logger.exception(f"no retry") + except Exception: + app.logger.exception(f"failed to cache attachments for {iri}") + abort(500) + + +def post_to_outbox(activity: ap.BaseActivity) -> str: + if activity.has_type(ap.CREATE_TYPES): + activity = activity.build_create() + + # Assign create a random ID + obj_id = back.random_object_id() + activity.set_id(back.activity_url(obj_id), obj_id) + + back.save(Box.OUTBOX, activity) + Tasks.cache_actor(activity.id) + Tasks.finish_post_to_outbox(activity.id) + return activity.id + + +def post_to_inbox(activity: ap.BaseActivity) -> None: + # Check for Block activity + actor = activity.get_actor() + if back.outbox_is_blocked(MY_PERSON, actor.id): + app.logger.info( + f"actor {actor!r} is blocked, dropping the received activity {activity!r}" + ) + return + + if back.inbox_check_duplicate(MY_PERSON, activity.id): + # The activity is already in the inbox + app.logger.info(f"received duplicate activity {activity!r}, dropping it") + + back.save(Box.INBOX, activity) + Tasks.process_new_activity(activity.id) + + app.logger.info(f"spawning task for {activity!r}") + Tasks.finish_post_to_inbox(activity.id) + + +def invalidate_cache(activity): + if activity.has_type(ap.ActivityType.LIKE): + if activity.get_object().id.startswith(BASE_URL): + DB.cache2.remove() + elif activity.has_type(ap.ActivityType.ANNOUNCE): + if activity.get_object().id.startswith(BASE_URL): + DB.cache2.remove() + elif activity.has_type(ap.ActivityType.UNDO): + DB.cache2.remove() + elif activity.has_type(ap.ActivityType.DELETE): + # TODO(tsileo): only invalidate if it's a delete of a reply + DB.cache2.remove() + elif activity.has_type(ap.ActivityType.UPDATE): + DB.cache2.remove() + elif activity.has_type(ap.ActivityType.CREATE): + note = activity.get_object() + if not note.inReplyTo or note.inReplyTo.startswith(ID): + DB.cache2.remove() + # FIXME(tsileo): check if it's a reply of a reply + + +@app.route("/task/cache_attachments", methods=["POST"]) +def task_cache_attachments(): + task = p.parse(request) + app.logger.info(f"task={task!r}") + iri = task.payload + try: + activity = ap.fetch_remote_activity(iri) + app.logger.info(f"activity={activity!r}") + # Generates thumbnails for the actor's icon and the attachments if any + + actor = activity.get_actor() + + # Update the cached actor + DB.actors.update_one( + {"remote_id": iri}, + {"$set": {"remote_id": iri, "data": actor.to_dict(embed=True)}}, + upsert=True, + ) + + if actor.icon: + MEDIA_CACHE.cache(actor.icon["url"], Kind.ACTOR_ICON) + + if activity.has_type(ap.ActivityType.CREATE): + for attachment in activity.get_object()._data.get("attachment", []): + if ( + attachment.get("mediaType", "").startswith("image/") + or attachment.get("type") == ap.ActivityType.IMAGE.value + ): + try: + MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT) + except ValueError: + app.logger.exception(f"failed to cache {attachment}") + + app.logger.info(f"attachments cached for {iri}") + + except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): + app.logger.exception(f"dropping activity {iri}, no attachment caching") + except Exception: + app.logger.exception(f"failed to cache attachments for {iri}") + abort(500) + + +@app.route("/task/cache_actor", methods=["POST"]) +def task_cache_actor(): + task = p.parse(request) + app.logger.info(f"task={task!r}") + iri, also_cache_attachments = ( + task.payload["iri"], + task.payload.get("also_cache_attachments", True), + ) + try: + activity = ap.fetch_remote_activity(iri) + app.logger.info(f"activity={activity!r}") + + if activity.has_type(ap.ActivityType.CREATE): + Tasks.fetch_og_metadata(iri) + + if activity.has_type([ap.ActivityType.LIKE, ap.ActivityType.ANNOUNCE]): + Tasks.cache_object(iri) + + actor = activity.get_actor() + + cache_actor_with_inbox = False + if activity.has_type(ap.ActivityType.FOLLOW): + if actor.id != ID: + # It's a Follow from the Inbox + cache_actor_with_inbox = True + else: + # It's a new following, cache the "object" (which is the actor we follow) + DB.activities.update_one( + {"remote_id": iri}, + { + "$set": { + "meta.object": activitypub._actor_to_meta( + activity.get_object() + ) + } + }, + ) + + # Cache the actor info + DB.activities.update_one( + {"remote_id": iri}, + { + "$set": { + "meta.actor": activitypub._actor_to_meta( + actor, cache_actor_with_inbox + ) + } + }, + ) + + app.logger.info(f"actor cached for {iri}") + if also_cache_attachments and activity.has_type(ap.ActivityType.CREATE): + Tasks.cache_attachments(iri) + + except (ActivityGoneError, ActivityNotFoundError): + DB.activities.update_one({"remote_id": iri}, {"$set": {"meta.deleted": True}}) + app.logger.exception(f"flagging activity {iri} as deleted, no actor caching") + except Exception: + app.logger.exception(f"failed to cache actor for {iri}") + abort(500) + + +@app.route("/task/process_new_activity", methods=["POST"]) # noqa:c901 +def task_process_new_activity(): + """Process an activity received in the inbox""" + task = p.parse(request) + app.logger.info(f"task={task!r}") + iri = task.payload + try: + activity = ap.fetch_remote_activity(iri) + app.logger.info(f"activity={activity!r}") + + # Is the activity expected? + # following = ap.get_backend().following() + should_forward = False + should_delete = False + + tag_stream = False + if activity.has_type(ap.ActivityType.ANNOUNCE): + try: + activity.get_object() + tag_stream = True + except (NotAnActivityError, BadActivityError): + app.logger.exception(f"failed to get announce object for {activity!r}") + # Most likely on OStatus notice + tag_stream = False + should_delete = True + except (ActivityGoneError, ActivityNotFoundError): + # The announced activity is deleted/gone, drop it + should_delete = True + + elif activity.has_type(ap.ActivityType.CREATE): + note = activity.get_object() + # Make the note part of the stream if it's not a reply, or if it's a local reply + if not note.inReplyTo or note.inReplyTo.startswith(ID): + tag_stream = True + + if note.inReplyTo: + try: + reply = ap.fetch_remote_activity(note.inReplyTo) + if ( + reply.id.startswith(ID) or reply.has_mention(ID) + ) and activity.is_public(): + # The reply is public "local reply", forward the reply (i.e. the original activity) to the + # original recipients + should_forward = True + except NotAnActivityError: + # Most likely a reply to an OStatus notce + should_delete = True + + # (partial) Ghost replies handling + # [X] This is the first time the server has seen this Activity. + should_forward = False + local_followers = ID + "/followers" + for field in ["to", "cc"]: + if field in activity._data: + if local_followers in activity._data[field]: + # [X] The values of to, cc, and/or audience contain a Collection owned by the server. + should_forward = True + + # [X] The values of inReplyTo, object, target and/or tag are objects owned by the server + if not (note.inReplyTo and note.inReplyTo.startswith(ID)): + should_forward = False + + elif activity.has_type(ap.ActivityType.DELETE): + note = DB.activities.find_one( + {"activity.object.id": activity.get_object().id} + ) + if note and note["meta"].get("forwarded", False): + # If the activity was originally forwarded, forward the delete too + should_forward = True + + elif activity.has_type(ap.ActivityType.LIKE): + if not activity.get_object_id().startswith(BASE_URL): + # We only want to keep a like if it's a like for a local activity + # (Pleroma relay the likes it received, we don't want to store them) + should_delete = True + + if should_forward: + app.logger.info(f"will forward {activity!r} to followers") + Tasks.forward_activity(activity.id) + + if should_delete: + app.logger.info(f"will soft delete {activity!r}") + + app.logger.info(f"{iri} tag_stream={tag_stream}") + DB.activities.update_one( + {"remote_id": activity.id}, + { + "$set": { + "meta.stream": tag_stream, + "meta.forwarded": should_forward, + "meta.deleted": should_delete, + } + }, + ) + + app.logger.info(f"new activity {iri} processed") + if not should_delete and not activity.has_type(ap.ActivityType.DELETE): + Tasks.cache_actor(iri) + except (ActivityGoneError, ActivityNotFoundError): + app.logger.log.exception(f"dropping activity {iri}, skip processing") + except Exception: + app.logger.exception(f"failed to process new activity {iri}") + abort(500) + + +@app.route("/task/forward_activity") +def task_forward_activity(): + task = p.parse(request) + app.logger.info(f"task={task!r}") + iri = task.payload + try: + activity = ap.fetch_remote_activity(iri) + recipients = back.followers_as_recipients() + app.logger.debug(f"Forwarding {activity!r} to {recipients}") + activity = ap.clean_activity(activity.to_dict()) + payload = json.dumps(activity) + for recp in recipients: + app.logger.debug(f"forwarding {activity!r} to {recp}") + Tasks.post_to_remote_inbox(payload, recp) + except Exception: + app.logger.exception("task failed") + abort(500) + + +@app.route("/task/post_to_remote_inbox") +def task_post_to_remote_inbox(): + task = p.parse(request) + app.logger.info(f"task={task!r}") + payload, to = task.payload["payload"], task.payload["to"] + try: + app.logger.info("payload=%s", payload) + app.logger.info("generating sig") + signed_payload = json.loads(payload) + + # Don't overwrite the signature if we're forwarding an activity + if "signature" not in signed_payload: + generate_signature(signed_payload, KEY) + + app.logger.info("to=%s", to) + resp = requests.post( + to, + data=json.dumps(signed_payload), + auth=SIG_AUTH, + headers={ + "Content-Type": HEADERS[1], + "Accept": HEADERS[1], + "User-Agent": USER_AGENT, + }, + ) + app.logger.info("resp=%s", resp) + app.logger.info("resp_body=%s", resp.text) + resp.raise_for_status() + except HTTPError as err: + app.logger.exception("request failed") + if 400 >= err.response.status_code >= 499: + app.logger.info("client error, no retry") + return "" + + abort(500) diff --git a/config.py b/config.py index b66ce71..846ae79 100644 --- a/config.py +++ b/config.py @@ -105,12 +105,17 @@ MEDIA_CACHE = MediaCache(GRIDFS, USER_AGENT) def create_indexes(): DB.activities.create_index([("remote_id", pymongo.ASCENDING)]) DB.activities.create_index([("activity.object.id", pymongo.ASCENDING)]) - DB.activities.create_index([ - ("activity.object.id", pymongo.ASCENDING), - ("meta.deleted", pymongo.ASCENDING), - ]) - DB.cache2.create_index([("path", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ("arg", pymongo.ASCENDING)]) - DB.cache2.create_index("date", expireAfterSeconds=3600*12) + DB.activities.create_index( + [("activity.object.id", pymongo.ASCENDING), ("meta.deleted", pymongo.ASCENDING)] + ) + DB.cache2.create_index( + [ + ("path", pymongo.ASCENDING), + ("type", pymongo.ASCENDING), + ("arg", pymongo.ASCENDING), + ] + ) + DB.cache2.create_index("date", expireAfterSeconds=3600 * 12) # Index for the block query DB.activities.create_index( diff --git a/docker-compose-tests.yml b/docker-compose-tests.yml index eae4264..0a9c393 100644 --- a/docker-compose-tests.yml +++ b/docker-compose-tests.yml @@ -4,9 +4,6 @@ services: image: 'microblogpub:latest' ports: - "${WEB_PORT}:5005" - links: - - mongo - - rmq volumes: - "${CONFIG_DIR}:/app/config" - "./static:/app/static" @@ -14,12 +11,10 @@ services: - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// - MICROBLOGPUB_MONGODB_HOST=mongo:27017 - MICROBLOGPUB_DEBUG=1 + - POUSSETACHES_AUTH_KEY=123 celery: # image: "instance1_web" image: 'microblogpub:latest' - links: - - mongo - - rmq command: 'celery worker -l info -A tasks' volumes: - "${CONFIG_DIR}:/app/config" @@ -35,6 +30,10 @@ services: environment: - RABBITMQ_ERLANG_COOKIE=secretrabbit - RABBITMQ_NODENAME=rabbit@my-rabbit + poussetaches: + image: "poussetaches:latest" + environment: + - POUSSETACHES_AUTH_KEY=123 networks: default: name: microblogpubfede diff --git a/docker-compose.yml b/docker-compose.yml index b7f9521..e77552f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,12 +7,14 @@ services: links: - mongo - rmq + - poussetaches volumes: - "${CONFIG_DIR}:/app/config" - "./static:/app/static" environment: - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// - MICROBLOGPUB_MONGODB_HOST=mongo:27017 + - POUSSETACHES_AUTH_KEY=123 celery: image: 'microblogpub:latest' links: @@ -36,3 +38,7 @@ services: - RABBITMQ_NODENAME=rabbit@my-rabbit volumes: - "${DATA_DIR}/rabbitmq:/var/lib/rabbitmq" + poussetaches: + image: "poussetaches:latest" + environment: + - POUSSETACHES_AUTH_KEY=123 diff --git a/poussetaches.py b/poussetaches.py new file mode 100644 index 0000000..f2532bc --- /dev/null +++ b/poussetaches.py @@ -0,0 +1,48 @@ +import base64 +import json +import os +from typing import Any +from dataclasses import dataclass +import flask +import requests + +POUSSETACHES_AUTH_KEY = os.getenv("POUSSETACHES_AUTH_KEY") + + +@dataclass +class Task: + req_id: str + tries: int + + payload: Any + + +class PousseTaches: + def __init__(self, api_url: str, base_url: str) -> None: + self.api_url = api_url + self.base_url = base_url + + def push(self, payload: Any, path: str, expected=200) -> str: + # Encode our payload + p = base64.b64encode(json.dumps(payload).encode()).decode() + + # Queue/push it + resp = requests.post( + self.api_url, + json={"url": self.base_url + path, "payload": p, "expected": expected}, + ) + resp.raise_for_status() + + return resp.headers.get("Poussetaches-Task-ID") + + def parse(self, req: flask.Request) -> Task: + if req.headers.get("Poussetaches-Auth-Key") != POUSSETACHES_AUTH_KEY: + raise ValueError("Bad auth key") + + # Parse the "envelope" + envelope = json.loads(req.data) + print(req) + print(f"envelope={envelope!r}") + payload = json.loads(base64.b64decode(envelope["payload"])) + + return Task(req_id=envelope["req_id"], tries=envelope["tries"], payload=payload) diff --git a/tasks.py b/tasks.py index 7a2f64d..f34d217 100644 --- a/tasks.py +++ b/tasks.py @@ -339,6 +339,7 @@ def invalidate_cache(activity): DB.cache2.remove() # FIXME(tsileo): check if it's a reply of a reply + @app.task(bind=True, max_retries=MAX_RETRIES) # noqa: C901 def finish_post_to_inbox(self, iri: str) -> None: try: From 2ed79e9e27825efb0f79da2bda13917b6ec7a0e4 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 11:42:14 +0200 Subject: [PATCH 0332/1425] Fix config --- docker-compose-tests.yml | 2 ++ docker-compose.yml | 2 ++ poussetaches.py | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docker-compose-tests.yml b/docker-compose-tests.yml index 0a9c393..5d93b0e 100644 --- a/docker-compose-tests.yml +++ b/docker-compose-tests.yml @@ -32,6 +32,8 @@ services: - RABBITMQ_NODENAME=rabbit@my-rabbit poussetaches: image: "poussetaches:latest" + ports: + - '7991' environment: - POUSSETACHES_AUTH_KEY=123 networks: diff --git a/docker-compose.yml b/docker-compose.yml index e77552f..bfb08a6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,5 +40,7 @@ services: - "${DATA_DIR}/rabbitmq:/var/lib/rabbitmq" poussetaches: image: "poussetaches:latest" + ports: + - '7991' environment: - POUSSETACHES_AUTH_KEY=123 diff --git a/poussetaches.py b/poussetaches.py index f2532bc..ea989b6 100644 --- a/poussetaches.py +++ b/poussetaches.py @@ -33,7 +33,7 @@ class PousseTaches: ) resp.raise_for_status() - return resp.headers.get("Poussetaches-Task-ID") + return resp.headers["Poussetaches-Task-ID"] def parse(self, req: flask.Request) -> Task: if req.headers.get("Poussetaches-Auth-Key") != POUSSETACHES_AUTH_KEY: From b7681b3a02e62e0e5da78c5b3b3473d45b58fcd9 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 15:14:57 +0200 Subject: [PATCH 0333/1425] Fix CI --- app.py | 26 +++++++++++++++++++++----- run.sh | 2 +- tests/federation_test.py | 3 ++- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/app.py b/app.py index c9ac063..34894a6 100644 --- a/app.py +++ b/app.py @@ -2252,7 +2252,7 @@ def task_t2(): @app.route("/task/fetch_og_meta", methods=["POST"]) -def task_fetch_og_metadata(): +def task_fetch_og_meta(): task = p.parse(request) app.logger.info(f"task={task!r}") iri = task.payload @@ -2287,6 +2287,8 @@ def task_fetch_og_metadata(): app.logger.exception(f"failed to fetch OG metadata for {iri}") abort(500) + return "" + @app.route("/task/cache_object", methods=["POST"]) def task_cache_object(): @@ -2405,6 +2407,8 @@ def task_finish_post_to_outbox(): app.logger.exception(f"failed to post to remote inbox for {iri}") abort(500) + return "" + @app.route("/task/finish_post_to_inbox", methods=["POST"]) # noqa: C901 def task_finish_post_to_inbox(): @@ -2447,6 +2451,8 @@ def task_finish_post_to_inbox(): app.logger.exception(f"failed to cache attachments for {iri}") abort(500) + return "" + def post_to_outbox(activity: ap.BaseActivity) -> str: if activity.has_type(ap.CREATE_TYPES): @@ -2544,9 +2550,11 @@ def task_cache_attachments(): app.logger.exception(f"failed to cache attachments for {iri}") abort(500) + return "" + @app.route("/task/cache_actor", methods=["POST"]) -def task_cache_actor(): +def task_cache_actor() -> str: task = p.parse(request) app.logger.info(f"task={task!r}") iri, also_cache_attachments = ( @@ -2558,7 +2566,7 @@ def task_cache_actor(): app.logger.info(f"activity={activity!r}") if activity.has_type(ap.ActivityType.CREATE): - Tasks.fetch_og_metadata(iri) + Tasks.fetch_og_meta(iri) if activity.has_type([ap.ActivityType.LIKE, ap.ActivityType.ANNOUNCE]): Tasks.cache_object(iri) @@ -2606,6 +2614,8 @@ def task_cache_actor(): app.logger.exception(f"failed to cache actor for {iri}") abort(500) + return "" + @app.route("/task/process_new_activity", methods=["POST"]) # noqa:c901 def task_process_new_activity(): @@ -2711,8 +2721,10 @@ def task_process_new_activity(): app.logger.exception(f"failed to process new activity {iri}") abort(500) + return "" -@app.route("/task/forward_activity") + +@app.route("/task/forward_activity", methods=["POST"]) def task_forward_activity(): task = p.parse(request) app.logger.info(f"task={task!r}") @@ -2730,8 +2742,10 @@ def task_forward_activity(): app.logger.exception("task failed") abort(500) + return "" -@app.route("/task/post_to_remote_inbox") + +@app.route("/task/post_to_remote_inbox", methods=["POST"]) def task_post_to_remote_inbox(): task = p.parse(request) app.logger.info(f"task={task!r}") @@ -2766,3 +2780,5 @@ def task_post_to_remote_inbox(): return "" abort(500) + + return "" diff --git a/run.sh b/run.sh index a29f5fc..a7eef03 100755 --- a/run.sh +++ b/run.sh @@ -1,3 +1,3 @@ #!/bin/bash python -c "import config; config.create_indexes()" -gunicorn -t 300 -w 2 -b 0.0.0.0:5005 --log-level debug app:app +gunicorn -t 300 -w 5 -b 0.0.0.0:5005 --log-level debug app:app diff --git a/tests/federation_test.py b/tests/federation_test.py index 20569bb..bdaf58d 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -19,7 +19,7 @@ class Instance(object): def __init__(self, name, host_url, docker_url=None): self.host_url = host_url self.docker_url = docker_url or host_url - self._create_delay = 12 + self._create_delay = 60 with open( os.path.join( os.path.dirname(os.path.abspath(__file__)), @@ -50,6 +50,7 @@ class Instance(object): def debug(self): """Returns the debug infos (number of items in the inbox/outbox.""" + time.sleep(self._create_delay) resp = requests.get( f"{self.host_url}/api/debug", headers={**self._auth_headers, "Accept": "application/json"}, From 61624b6e75c5693ca4a04a1d3b00277c2696ed26 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 20:03:49 +0200 Subject: [PATCH 0334/1425] Fix tests --- app.py | 3 ++- docker-compose-tests.yml | 3 ++- poussetaches.py | 2 +- tests/federation_test.py | 3 +-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app.py b/app.py index 34894a6..4e3ef0b 100644 --- a/app.py +++ b/app.py @@ -88,7 +88,8 @@ from utils.media import Kind from poussetaches import PousseTaches -p = PousseTaches("http://poussetaches:7991", "http://web:5005") +phost = "http://" + os.getenv("COMPOSE_PROJECT_NAME", "") +p = PousseTaches(f"{phost}_poussetaches_1:7991", f"{phost}_web_1:5005") back = activitypub.MicroblogPubBackend() diff --git a/docker-compose-tests.yml b/docker-compose-tests.yml index 5d93b0e..fc94f79 100644 --- a/docker-compose-tests.yml +++ b/docker-compose-tests.yml @@ -9,8 +9,9 @@ services: - "./static:/app/static" environment: - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// - - MICROBLOGPUB_MONGODB_HOST=mongo:27017 + - MICROBLOGPUB_MONGODB_HOST=${COMPOSE_PROJECT_NAME}_mongo_1:27017 - MICROBLOGPUB_DEBUG=1 + - COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME} - POUSSETACHES_AUTH_KEY=123 celery: # image: "instance1_web" diff --git a/poussetaches.py b/poussetaches.py index ea989b6..6bf9180 100644 --- a/poussetaches.py +++ b/poussetaches.py @@ -45,4 +45,4 @@ class PousseTaches: print(f"envelope={envelope!r}") payload = json.loads(base64.b64decode(envelope["payload"])) - return Task(req_id=envelope["req_id"], tries=envelope["tries"], payload=payload) + return Task(req_id=envelope["req_id"], tries=envelope["tries"], payload=payload) # type: ignore diff --git a/tests/federation_test.py b/tests/federation_test.py index bdaf58d..959280f 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -19,7 +19,7 @@ class Instance(object): def __init__(self, name, host_url, docker_url=None): self.host_url = host_url self.docker_url = docker_url or host_url - self._create_delay = 60 + self._create_delay = 15 with open( os.path.join( os.path.dirname(os.path.abspath(__file__)), @@ -50,7 +50,6 @@ class Instance(object): def debug(self): """Returns the debug infos (number of items in the inbox/outbox.""" - time.sleep(self._create_delay) resp = requests.get( f"{self.host_url}/api/debug", headers={**self._auth_headers, "Accept": "application/json"}, From 1147ec80537e23555af8b931680a271d6a807d7d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 20:06:18 +0200 Subject: [PATCH 0335/1425] Fix CI --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index db184e9..f9fc688 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,9 +24,9 @@ script: - docker build . -t microblogpub:latest - docker-compose up -d - docker-compose ps - - WEB_PORT=5006 CONFIG_DIR=./tests/fixtures/instance1/config docker-compose -p instance1 -f docker-compose-tests.yml up -d + - WEB_PORT=5006 COMPOSE_PROJECT_NAME=instance1 CONFIG_DIR=./tests/fixtures/instance1/config docker-compose -p instance1 -f docker-compose-tests.yml up -d - docker-compose -p instance1 -f docker-compose-tests.yml ps - - WEB_PORT=5007 CONFIG_DIR=./tests/fixtures/instance2/config docker-compose -p instance2 -f docker-compose-tests.yml up -d + - WEB_PORT=5007 COMPOSE_PROJECT_NAME=instance2 CONFIG_DIR=./tests/fixtures/instance2/config docker-compose -p instance2 -f docker-compose-tests.yml up -d - docker-compose -p instance2 -f docker-compose-tests.yml ps - sleep 5 # Integration tests first From 8c3eedac7d10fc028b54bf54895cd75ea0a89fed Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 20:08:12 +0200 Subject: [PATCH 0336/1425] Re-enable Celery for the migration --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 4e3ef0b..46d3753 100644 --- a/app.py +++ b/app.py @@ -62,7 +62,7 @@ from werkzeug.utils import secure_filename import activitypub import config -# import tasks +import tasks from activitypub import Box from activitypub import embed_collection from config import USER_AGENT From 5d8fa38d5e2754d41a720aba79e58bd1ba3a1e92 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 21:36:56 +0200 Subject: [PATCH 0337/1425] More poussetaches integrations --- app.py | 102 +++++++++++++++++---------------------- docker-compose-tests.yml | 1 - docker-compose.yml | 8 +-- poussetaches.py | 61 +++++++++++++++++++++++ 4 files changed, 110 insertions(+), 62 deletions(-) diff --git a/app.py b/app.py index 46d3753..e320d83 100644 --- a/app.py +++ b/app.py @@ -62,7 +62,7 @@ from werkzeug.utils import secure_filename import activitypub import config -import tasks +import tasks # noqa: here just for the migration # FIXME(tsileo): remove me from activitypub import Box from activitypub import embed_collection from config import USER_AGENT @@ -2210,6 +2210,9 @@ def token_endpoint(): ) +################# +# Feeds + @app.route("/feed.json") def json_feed(): return Response( @@ -2234,22 +2237,48 @@ def rss_feed(): ) -@app.route("/task/t1") -def task_t1(): - p.push( - "https://mastodon.cloud/@iulius/101852467780804071/activity", - "/task/cache_object", - ) - return "ok" +########### +# Tasks +class Tasks: + @staticmethod + def cache_object(iri: str) -> None: + p.push(iri, "/task/cache_object") -@app.route("/task/t2", methods=["POST"]) -def task_t2(): - print(request) - print(request.headers) - task = p.parse(request) - print(task) - return "yay" + @staticmethod + def cache_actor(iri: str, also_cache_attachments: bool = True) -> None: + p.push( + {"iri": iri, "also_cache_attachments": also_cache_attachments}, + "/task/cache_actor", + ) + + @staticmethod + def post_to_remote_inbox(payload: str, recp: str) -> None: + p.push({"payload": payload, "to": recp}, "/task/post_to_remote_inbox") + + @staticmethod + def forward_activity(iri: str) -> None: + p.push(iri, "/task/forward_activity") + + @staticmethod + def fetch_og_meta(iri: str) -> None: + p.push(iri, "/task/fetch_og_meta") + + @staticmethod + def process_new_activity(iri: str) -> None: + p.push(iri, "/task/process_new_activity") + + @staticmethod + def cache_attachments(iri: str) -> None: + p.push(iri, "/task/cache_attachments") + + @staticmethod + def finish_post_to_inbox(iri: str) -> None: + p.push(iri, "/task/finish_post_to_inbox") + + @staticmethod + def finish_post_to_outbox(iri: str) -> None: + p.push(iri, "/task/finish_post_to_outbox") @app.route("/task/fetch_og_meta", methods=["POST"]) @@ -2321,48 +2350,6 @@ def task_cache_object(): abort(500) return "" - -class Tasks: - @staticmethod - def cache_object(iri: str) -> None: - p.push(iri, "/task/cache_object") - - @staticmethod - def cache_actor(iri: str, also_cache_attachments: bool = True) -> None: - p.push( - {"iri": iri, "also_cache_attachments": also_cache_attachments}, - "/task/cache_actor", - ) - - @staticmethod - def post_to_remote_inbox(payload: str, recp: str) -> None: - p.push({"payload": payload, "to": recp}, "/task/post_to_remote_inbox") - - @staticmethod - def forward_activity(iri: str) -> None: - p.push(iri, "/task/forward_activity") - - @staticmethod - def fetch_og_meta(iri: str) -> None: - p.push(iri, "/task/fetch_og_meta") - - @staticmethod - def process_new_activity(iri: str) -> None: - p.push(iri, "/task/process_new_activity") - - @staticmethod - def cache_attachments(iri: str) -> None: - p.push(iri, "/task/cache_attachments") - - @staticmethod - def finish_post_to_inbox(iri: str) -> None: - p.push(iri, "/task/finish_post_to_inbox") - - @staticmethod - def finish_post_to_outbox(iri: str) -> None: - p.push(iri, "/task/finish_post_to_outbox") - - @app.route("/task/finish_post_to_outbox", methods=["POST"]) # noqa:C901 def task_finish_post_to_outbox(): task = p.parse(request) @@ -2748,6 +2735,7 @@ def task_forward_activity(): @app.route("/task/post_to_remote_inbox", methods=["POST"]) def task_post_to_remote_inbox(): + """Post an activity to a remote inbox.""" task = p.parse(request) app.logger.info(f"task={task!r}") payload, to = task.payload["payload"], task.payload["to"] diff --git a/docker-compose-tests.yml b/docker-compose-tests.yml index fc94f79..3360bdc 100644 --- a/docker-compose-tests.yml +++ b/docker-compose-tests.yml @@ -33,7 +33,6 @@ services: - RABBITMQ_NODENAME=rabbit@my-rabbit poussetaches: image: "poussetaches:latest" - ports: - '7991' environment: - POUSSETACHES_AUTH_KEY=123 diff --git a/docker-compose.yml b/docker-compose.yml index bfb08a6..94c49ef 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: environment: - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// - MICROBLOGPUB_MONGODB_HOST=mongo:27017 - - POUSSETACHES_AUTH_KEY=123 + - POUSSETACHES_AUTH_KEY=${POUSSETACHES_AUTH_KEY} celery: image: 'microblogpub:latest' links: @@ -40,7 +40,7 @@ services: - "${DATA_DIR}/rabbitmq:/var/lib/rabbitmq" poussetaches: image: "poussetaches:latest" - ports: - - '7991' + volumes: + - "${DATA_DIR}/poussetaches:/app/poussetaches/poussetaches_data" environment: - - POUSSETACHES_AUTH_KEY=123 + - POUSSETACHES_AUTH_KEY=${POUSSETACHES_AUTH_KEY} diff --git a/poussetaches.py b/poussetaches.py index 6bf9180..cabf5ca 100644 --- a/poussetaches.py +++ b/poussetaches.py @@ -1,10 +1,13 @@ import base64 import json import os +from typing import Dict from typing import Any +from typing import List from dataclasses import dataclass import flask import requests +from datetime import datetime POUSSETACHES_AUTH_KEY = os.getenv("POUSSETACHES_AUTH_KEY") @@ -17,6 +20,18 @@ class Task: payload: Any +@dataclass +class GetTask: + payload: Any + expected: int + task_id: str + next_run: datetime + tries: int + url: str + last_error_status_code: int + last_error_body: str + + class PousseTaches: def __init__(self, api_url: str, base_url: str) -> None: self.api_url = api_url @@ -46,3 +61,49 @@ class PousseTaches: payload = json.loads(base64.b64decode(envelope["payload"])) return Task(req_id=envelope["req_id"], tries=envelope["tries"], payload=payload) # type: ignore + + @staticmethod + def _expand_task(t: Dict[str, Any]) -> None: + try: + t["payload"] = json.loads(base64.b64decode(t["payload"])) + except json.JSONDecodeError: + t["payload"] = base64.b64decode(t["payload"]).decode() + + if t["last_error_body"]: + t["last_error_body"] = base64.b64decode(t["last_error_body"]).decode() + + t["next_run"] = datetime.fromtimestamp(float(t["next_run"] / 1e9)) + if t["last_run"]: + t["last_run"] = datetime.fromtimestamp(float(t["last__run"] / 1e9)) + else: + del t["last_run"] + + def _get(self, where: str) -> List[GetTask]: + out = [] + + resp = requests.get(self.api_url + f"/{where}") + resp.raise_for_status() + dat = resp.json() + for t in dat["tasks"]: + self._expand_task(t) + out.append(GetTask( + task_id=t["id"], + payload=t["payload"], + expected=t["expected"], + tries=t["tries"], + url=t["url"], + last_error_status_code=t["last_error_status_code"], + last_error_body=t["last_error_body"], + next_run=t["next_run"], + )) + + return out + + def get_success(self) -> List[GetTask]: + return self._get("success") + + def get_waiting(self) -> List[GetTask]: + return self._get("waiting") + + def get_dead(self) -> List[GetTask]: + return self._get("dead") From 26125a0816a02c74c01c64745ccdbc85a7add609 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 21:42:44 +0200 Subject: [PATCH 0338/1425] Add data dir for tasks --- data/poussetaches/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 data/poussetaches/.gitignore diff --git a/data/poussetaches/.gitignore b/data/poussetaches/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/data/poussetaches/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore From ccfa2f0d890800c542eb07e59d7830f3fdfa7477 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 22:11:32 +0200 Subject: [PATCH 0339/1425] Fix docker compose config --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 94c49ef..23ba974 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,6 +41,6 @@ services: poussetaches: image: "poussetaches:latest" volumes: - - "${DATA_DIR}/poussetaches:/app/poussetaches/poussetaches_data" + - "${DATA_DIR}/poussetaches:/app/poussetaches_data" environment: - POUSSETACHES_AUTH_KEY=${POUSSETACHES_AUTH_KEY} From 2ef968f11e45534f81c9ae4fbc96819ec6146dea Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 22:16:10 +0200 Subject: [PATCH 0340/1425] Fix docker compose config --- docker-compose-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose-tests.yml b/docker-compose-tests.yml index 3360bdc..4d9443d 100644 --- a/docker-compose-tests.yml +++ b/docker-compose-tests.yml @@ -33,7 +33,6 @@ services: - RABBITMQ_NODENAME=rabbit@my-rabbit poussetaches: image: "poussetaches:latest" - - '7991' environment: - POUSSETACHES_AUTH_KEY=123 networks: From 2bd06886eca48f587cd37d9499cbd9789010b996 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 22:41:25 +0200 Subject: [PATCH 0341/1425] Fix compose config --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 23ba974..634987e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,7 @@ services: - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// - MICROBLOGPUB_MONGODB_HOST=mongo:27017 - POUSSETACHES_AUTH_KEY=${POUSSETACHES_AUTH_KEY} + - COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME} celery: image: 'microblogpub:latest' links: From 3289e91786e8bbef99d90e101090e156f1ab03ae Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 11:35:48 +0200 Subject: [PATCH 0342/1425] Try poussetaches --- .travis.yml | 1 + app.py | 646 ++++++++++++++++++++++++++++++++++++--- config.py | 17 +- docker-compose-tests.yml | 11 +- docker-compose.yml | 6 + poussetaches.py | 48 +++ tasks.py | 1 + 7 files changed, 673 insertions(+), 57 deletions(-) create mode 100644 poussetaches.py diff --git a/.travis.yml b/.travis.yml index 2e8ab86..db184e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ install: - sudo chmod +x /usr/local/bin/docker-compose - docker-compose --version - pip install -r dev-requirements.txt + - git clone https://github.com/tsileo/poussetaches.git && cd poussetaches && docker build . -t poussetaches:latest && cd - script: - mypy --ignore-missing-imports . - flake8 activitypub.py diff --git a/app.py b/app.py index 0730813..c9ac063 100644 --- a/app.py +++ b/app.py @@ -16,6 +16,8 @@ from typing import Tuple from urllib.parse import urlencode from urllib.parse import urlparse +from requests.exceptions import HTTPError +import requests import bleach import mf2py import pymongo @@ -41,7 +43,10 @@ from little_boxes.activitypub import _to_list from little_boxes.activitypub import clean_activity from little_boxes.activitypub import get_backend from little_boxes.content_helper import parse_markdown +from little_boxes.linked_data_sig import generate_signature from little_boxes.errors import ActivityGoneError +from little_boxes.errors import NotAnActivityError +from little_boxes.errors import BadActivityError from little_boxes.errors import ActivityNotFoundError from little_boxes.errors import Error from little_boxes.errors import NotFromOutboxError @@ -49,15 +54,18 @@ from little_boxes.httpsig import HTTPSigAuth from little_boxes.httpsig import verify_request from little_boxes.webfinger import get_actor_url from little_boxes.webfinger import get_remote_follow_template +from utils import opengraph from passlib.hash import bcrypt from u2flib_server import u2f from werkzeug.utils import secure_filename import activitypub import config -import tasks + +# import tasks from activitypub import Box from activitypub import embed_collection +from config import USER_AGENT from config import ADMIN_API_KEY from config import BASE_URL from config import DB @@ -78,6 +86,11 @@ from utils.key import get_secret_key from utils.lookup import lookup from utils.media import Kind +from poussetaches import PousseTaches + +p = PousseTaches("http://poussetaches:7991", "http://web:5005") + + back = activitypub.MicroblogPubBackend() ap.use_backend(back) @@ -191,7 +204,7 @@ ALLOWED_TAGS = [ def clean_html(html): try: return bleach.clean(html, tags=ALLOWED_TAGS) - except: + except Exception: return "" @@ -631,7 +644,7 @@ def authorize_follow(): return redirect("/following") follow = ap.Follow(actor=MY_PERSON.id, object=actor) - tasks.post_to_outbox(follow) + post_to_outbox(follow) return redirect("/following") @@ -758,7 +771,7 @@ def tmp_migrate4(): @login_required def tmp_migrate5(): for activity in DB.activities.find(): - tasks.cache_actor.delay(activity["remote_id"], also_cache_attachments=False) + Tasks.cache_actor(activity["remote_id"], also_cache_attachments=False) return "Done" @@ -835,9 +848,10 @@ def _get_cached(type_="html", arg=None): cached = DB.cache2.find_one({"path": request.path, "type": type_, "arg": arg}) if cached: app.logger.info("from cache") - return cached['response_data'] + return cached["response_data"] return None + def _cache(resp, type_="html", arg=None): if not CACHING: return None @@ -855,7 +869,9 @@ def _cache(resp, type_="html", arg=None): def index(): if is_api_request(): return jsonify(**ME) - cache_arg = f"{request.args.get('older_than', '')}:{request.args.get('newer_than', '')}" + cache_arg = ( + f"{request.args.get('older_than', '')}:{request.args.get('newer_than', '')}" + ) cached = _get_cached("html", cache_arg) if cached: return cached @@ -1053,22 +1069,22 @@ def nodeinfo(): } response = json.dumps( - { - "version": "2.0", - "software": { - "name": "microblogpub", - "version": f"Microblog.pub {VERSION}", - }, - "protocols": ["activitypub"], - "services": {"inbound": [], "outbound": []}, - "openRegistrations": False, - "usage": {"users": {"total": 1}, "localPosts": DB.activities.count(q)}, - "metadata": { - "sourceCode": "https://github.com/tsileo/microblog.pub", - "nodeName": f"@{USERNAME}@{DOMAIN}", - }, - } - ) + { + "version": "2.0", + "software": { + "name": "microblogpub", + "version": f"Microblog.pub {VERSION}", + }, + "protocols": ["activitypub"], + "services": {"inbound": [], "outbound": []}, + "openRegistrations": False, + "usage": {"users": {"total": 1}, "localPosts": DB.activities.count(q)}, + "metadata": { + "sourceCode": "https://github.com/tsileo/microblog.pub", + "nodeName": f"@{USERNAME}@{DOMAIN}", + }, + } + ) if not cached: _cache(response, "api") @@ -1197,7 +1213,7 @@ def outbox(): data = request.get_json(force=True) print(data) activity = ap.parse_activity(data) - activity_id = tasks.post_to_outbox(activity) + activity_id = post_to_outbox(activity) return Response(status=201, headers={"Location": activity_id}) @@ -1536,11 +1552,15 @@ def _user_api_get_note(from_outbox: bool = False): oid = _user_api_arg("id") app.logger.info(f"fetching {oid}") try: - note = ap.parse_activity(get_backend().fetch_iri(oid), expected=ActivityType.NOTE) - except: + note = ap.parse_activity( + get_backend().fetch_iri(oid), expected=ActivityType.NOTE + ) + except Exception: try: - note = ap.parse_activity(get_backend().fetch_iri(oid), expected=ActivityType.VIDEO) - except: + note = ap.parse_activity( + get_backend().fetch_iri(oid), expected=ActivityType.VIDEO + ) + except Exception: raise ActivityNotFoundError( "Expected Note or Video ActivityType, but got something else" ) @@ -1570,7 +1590,7 @@ def api_delete(): delete = ap.Delete(actor=ID, object=ap.Tombstone(id=note.id).to_dict(embed=True)) - delete_id = tasks.post_to_outbox(delete) + delete_id = post_to_outbox(delete) return _user_api_response(activity=delete_id) @@ -1581,7 +1601,7 @@ def api_boost(): note = _user_api_get_note() announce = note.build_announce(MY_PERSON) - announce_id = tasks.post_to_outbox(announce) + announce_id = post_to_outbox(announce) return _user_api_response(activity=announce_id) @@ -1592,7 +1612,7 @@ def api_like(): note = _user_api_get_note() like = note.build_like(MY_PERSON) - like_id = tasks.post_to_outbox(like) + like_id = post_to_outbox(like) return _user_api_response(activity=like_id) @@ -1639,7 +1659,7 @@ def api_undo(): obj = ap.parse_activity(doc.get("activity")) # FIXME(tsileo): detect already undo-ed and make this API call idempotent undo = obj.build_undo() - undo_id = tasks.post_to_outbox(undo) + undo_id = post_to_outbox(undo) return _user_api_response(activity=undo_id) @@ -1664,7 +1684,7 @@ def admin_stream(): ) -@app.route("/inbox", methods=["GET", "POST"]) +@app.route("/inbox", methods=["GET", "POST"]) # noqa: C901 def inbox(): if request.method == "GET": if not is_api_request(): @@ -1733,7 +1753,7 @@ def inbox(): ) activity = ap.parse_activity(data) logger.debug(f"inbox activity={activity}/{data}") - tasks.post_to_inbox(activity) + post_to_inbox(activity) return Response(status=201) @@ -1819,7 +1839,7 @@ def api_new_note(): note = ap.Note(**raw_note) create = note.build_create() - create_id = tasks.post_to_outbox(create) + create_id = post_to_outbox(create) return _user_api_response(activity=create_id) @@ -1852,7 +1872,7 @@ def api_block(): return _user_api_response(activity=existing["activity"]["id"]) block = ap.Block(actor=MY_PERSON.id, object=actor) - block_id = tasks.post_to_outbox(block) + block_id = post_to_outbox(block) return _user_api_response(activity=block_id) @@ -1874,7 +1894,7 @@ def api_follow(): return _user_api_response(activity=existing["activity"]["id"]) follow = ap.Follow(actor=MY_PERSON.id, object=actor) - follow_id = tasks.post_to_outbox(follow) + follow_id = post_to_outbox(follow) return _user_api_response(activity=follow_id) @@ -1895,8 +1915,9 @@ def followers(): ) raw_followers, older_than, newer_than = paginated_query(DB.activities, q) - followers = [doc["meta"]["actor"] - for doc in raw_followers if "actor" in doc.get("meta", {})] + followers = [ + doc["meta"]["actor"] for doc in raw_followers if "actor" in doc.get("meta", {}) + ] return render_template( "followers.html", followers_data=followers, @@ -1924,9 +1945,11 @@ def following(): abort(404) following, older_than, newer_than = paginated_query(DB.activities, q) - following = [(doc["remote_id"], doc["meta"]["object"]) - for doc in following - if "remote_id" in doc and "object" in doc.get("meta", {})] + following = [ + (doc["remote_id"], doc["meta"]["object"]) + for doc in following + if "remote_id" in doc and "object" in doc.get("meta", {}) + ] return render_template( "following.html", following_data=following, @@ -2087,7 +2110,7 @@ def indieauth_flow(): return redirect(red) -@app.route('/indieauth', methods=['GET', 'POST']) +@app.route("/indieauth", methods=["GET", "POST"]) def indieauth_endpoint(): if request.method == "GET": if not session.get("logged_in"): @@ -2189,9 +2212,7 @@ def token_endpoint(): @app.route("/feed.json") def json_feed(): return Response( - response=json.dumps( - activitypub.json_feed("/feed.json") - ), + response=json.dumps(activitypub.json_feed("/feed.json")), headers={"Content-Type": "application/json"}, ) @@ -2210,3 +2231,538 @@ def rss_feed(): response=activitypub.gen_feed().rss_str(), headers={"Content-Type": "application/rss+xml"}, ) + + +@app.route("/task/t1") +def task_t1(): + p.push( + "https://mastodon.cloud/@iulius/101852467780804071/activity", + "/task/cache_object", + ) + return "ok" + + +@app.route("/task/t2", methods=["POST"]) +def task_t2(): + print(request) + print(request.headers) + task = p.parse(request) + print(task) + return "yay" + + +@app.route("/task/fetch_og_meta", methods=["POST"]) +def task_fetch_og_metadata(): + task = p.parse(request) + app.logger.info(f"task={task!r}") + iri = task.payload + try: + activity = ap.fetch_remote_activity(iri) + app.logger.info(f"activity={activity!r}") + if activity.has_type(ap.ActivityType.CREATE): + note = activity.get_object() + links = opengraph.links_from_note(note.to_dict()) + og_metadata = opengraph.fetch_og_metadata(USER_AGENT, links) + for og in og_metadata: + if not og.get("image"): + continue + MEDIA_CACHE.cache_og_image(og["image"]) + + app.logger.debug(f"OG metadata {og_metadata!r}") + DB.activities.update_one( + {"remote_id": iri}, {"$set": {"meta.og_metadata": og_metadata}} + ) + + app.logger.info(f"OG metadata fetched for {iri}") + except (ActivityGoneError, ActivityNotFoundError): + app.logger.exception(f"dropping activity {iri}, skip OG metedata") + return "" + except requests.exceptions.HTTPError as http_err: + if 400 <= http_err.response.status_code < 500: + app.logger.exception("bad request, no retry") + return "" + app.logger.exception("failed to fetch OG metadata") + abort(500) + except Exception: + app.logger.exception(f"failed to fetch OG metadata for {iri}") + abort(500) + + +@app.route("/task/cache_object", methods=["POST"]) +def task_cache_object(): + task = p.parse(request) + app.logger.info(f"task={task!r}") + iri = task.payload + try: + activity = ap.fetch_remote_activity(iri) + print(activity) + print(activity.__dict__) + app.logger.info(f"activity={activity!r}") + obj = activity + # obj = activity.get_object() + DB.activities.update_one( + {"remote_id": activity.id}, + { + "$set": { + "meta.object": obj.to_dict(embed=True), + "meta.object_actor": activitypub._actor_to_meta(obj.get_actor()), + } + }, + ) + except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): + DB.activities.update_one({"remote_id": iri}, {"$set": {"meta.deleted": True}}) + app.logger.exception(f"flagging activity {iri} as deleted, no object caching") + return "" + except Exception: + app.logger.exception(f"failed to cache object for {iri}") + abort(500) + return "" + + +class Tasks: + @staticmethod + def cache_object(iri: str) -> None: + p.push(iri, "/task/cache_object") + + @staticmethod + def cache_actor(iri: str, also_cache_attachments: bool = True) -> None: + p.push( + {"iri": iri, "also_cache_attachments": also_cache_attachments}, + "/task/cache_actor", + ) + + @staticmethod + def post_to_remote_inbox(payload: str, recp: str) -> None: + p.push({"payload": payload, "to": recp}, "/task/post_to_remote_inbox") + + @staticmethod + def forward_activity(iri: str) -> None: + p.push(iri, "/task/forward_activity") + + @staticmethod + def fetch_og_meta(iri: str) -> None: + p.push(iri, "/task/fetch_og_meta") + + @staticmethod + def process_new_activity(iri: str) -> None: + p.push(iri, "/task/process_new_activity") + + @staticmethod + def cache_attachments(iri: str) -> None: + p.push(iri, "/task/cache_attachments") + + @staticmethod + def finish_post_to_inbox(iri: str) -> None: + p.push(iri, "/task/finish_post_to_inbox") + + @staticmethod + def finish_post_to_outbox(iri: str) -> None: + p.push(iri, "/task/finish_post_to_outbox") + + +@app.route("/task/finish_post_to_outbox", methods=["POST"]) # noqa:C901 +def task_finish_post_to_outbox(): + task = p.parse(request) + app.logger.info(f"task={task!r}") + iri = task.payload + try: + activity = ap.fetch_remote_activity(iri) + app.logger.info(f"activity={activity!r}") + + recipients = activity.recipients() + + if activity.has_type(ap.ActivityType.DELETE): + back.outbox_delete(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.UPDATE): + back.outbox_update(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.CREATE): + back.outbox_create(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.ANNOUNCE): + back.outbox_announce(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.LIKE): + back.outbox_like(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.UNDO): + obj = activity.get_object() + if obj.has_type(ap.ActivityType.LIKE): + back.outbox_undo_like(MY_PERSON, obj) + elif obj.has_type(ap.ActivityType.ANNOUNCE): + back.outbox_undo_announce(MY_PERSON, obj) + elif obj.has_type(ap.ActivityType.FOLLOW): + back.undo_new_following(MY_PERSON, obj) + + app.logger.info(f"recipients={recipients}") + activity = ap.clean_activity(activity.to_dict()) + + DB.cache2.remove() + + payload = json.dumps(activity) + for recp in recipients: + app.logger.debug(f"posting to {recp}") + Tasks.post_to_remote_inbox(payload, recp) + except (ActivityGoneError, ActivityNotFoundError): + app.logger.exception(f"no retry") + except Exception: + app.logger.exception(f"failed to post to remote inbox for {iri}") + abort(500) + + +@app.route("/task/finish_post_to_inbox", methods=["POST"]) # noqa: C901 +def task_finish_post_to_inbox(): + task = p.parse(request) + app.logger.info(f"task={task!r}") + iri = task.payload + try: + activity = ap.fetch_remote_activity(iri) + app.logger.info(f"activity={activity!r}") + + if activity.has_type(ap.ActivityType.DELETE): + back.inbox_delete(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.UPDATE): + back.inbox_update(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.CREATE): + back.inbox_create(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.ANNOUNCE): + back.inbox_announce(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.LIKE): + back.inbox_like(MY_PERSON, activity) + elif activity.has_type(ap.ActivityType.FOLLOW): + # Reply to a Follow with an Accept + accept = ap.Accept(actor=ID, object=activity.to_dict(embed=True)) + post_to_outbox(accept) + elif activity.has_type(ap.ActivityType.UNDO): + obj = activity.get_object() + if obj.has_type(ap.ActivityType.LIKE): + back.inbox_undo_like(MY_PERSON, obj) + elif obj.has_type(ap.ActivityType.ANNOUNCE): + back.inbox_undo_announce(MY_PERSON, obj) + elif obj.has_type(ap.ActivityType.FOLLOW): + back.undo_new_follower(MY_PERSON, obj) + try: + invalidate_cache(activity) + except Exception: + app.logger.exception("failed to invalidate cache") + except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): + app.logger.exception(f"no retry") + except Exception: + app.logger.exception(f"failed to cache attachments for {iri}") + abort(500) + + +def post_to_outbox(activity: ap.BaseActivity) -> str: + if activity.has_type(ap.CREATE_TYPES): + activity = activity.build_create() + + # Assign create a random ID + obj_id = back.random_object_id() + activity.set_id(back.activity_url(obj_id), obj_id) + + back.save(Box.OUTBOX, activity) + Tasks.cache_actor(activity.id) + Tasks.finish_post_to_outbox(activity.id) + return activity.id + + +def post_to_inbox(activity: ap.BaseActivity) -> None: + # Check for Block activity + actor = activity.get_actor() + if back.outbox_is_blocked(MY_PERSON, actor.id): + app.logger.info( + f"actor {actor!r} is blocked, dropping the received activity {activity!r}" + ) + return + + if back.inbox_check_duplicate(MY_PERSON, activity.id): + # The activity is already in the inbox + app.logger.info(f"received duplicate activity {activity!r}, dropping it") + + back.save(Box.INBOX, activity) + Tasks.process_new_activity(activity.id) + + app.logger.info(f"spawning task for {activity!r}") + Tasks.finish_post_to_inbox(activity.id) + + +def invalidate_cache(activity): + if activity.has_type(ap.ActivityType.LIKE): + if activity.get_object().id.startswith(BASE_URL): + DB.cache2.remove() + elif activity.has_type(ap.ActivityType.ANNOUNCE): + if activity.get_object().id.startswith(BASE_URL): + DB.cache2.remove() + elif activity.has_type(ap.ActivityType.UNDO): + DB.cache2.remove() + elif activity.has_type(ap.ActivityType.DELETE): + # TODO(tsileo): only invalidate if it's a delete of a reply + DB.cache2.remove() + elif activity.has_type(ap.ActivityType.UPDATE): + DB.cache2.remove() + elif activity.has_type(ap.ActivityType.CREATE): + note = activity.get_object() + if not note.inReplyTo or note.inReplyTo.startswith(ID): + DB.cache2.remove() + # FIXME(tsileo): check if it's a reply of a reply + + +@app.route("/task/cache_attachments", methods=["POST"]) +def task_cache_attachments(): + task = p.parse(request) + app.logger.info(f"task={task!r}") + iri = task.payload + try: + activity = ap.fetch_remote_activity(iri) + app.logger.info(f"activity={activity!r}") + # Generates thumbnails for the actor's icon and the attachments if any + + actor = activity.get_actor() + + # Update the cached actor + DB.actors.update_one( + {"remote_id": iri}, + {"$set": {"remote_id": iri, "data": actor.to_dict(embed=True)}}, + upsert=True, + ) + + if actor.icon: + MEDIA_CACHE.cache(actor.icon["url"], Kind.ACTOR_ICON) + + if activity.has_type(ap.ActivityType.CREATE): + for attachment in activity.get_object()._data.get("attachment", []): + if ( + attachment.get("mediaType", "").startswith("image/") + or attachment.get("type") == ap.ActivityType.IMAGE.value + ): + try: + MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT) + except ValueError: + app.logger.exception(f"failed to cache {attachment}") + + app.logger.info(f"attachments cached for {iri}") + + except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): + app.logger.exception(f"dropping activity {iri}, no attachment caching") + except Exception: + app.logger.exception(f"failed to cache attachments for {iri}") + abort(500) + + +@app.route("/task/cache_actor", methods=["POST"]) +def task_cache_actor(): + task = p.parse(request) + app.logger.info(f"task={task!r}") + iri, also_cache_attachments = ( + task.payload["iri"], + task.payload.get("also_cache_attachments", True), + ) + try: + activity = ap.fetch_remote_activity(iri) + app.logger.info(f"activity={activity!r}") + + if activity.has_type(ap.ActivityType.CREATE): + Tasks.fetch_og_metadata(iri) + + if activity.has_type([ap.ActivityType.LIKE, ap.ActivityType.ANNOUNCE]): + Tasks.cache_object(iri) + + actor = activity.get_actor() + + cache_actor_with_inbox = False + if activity.has_type(ap.ActivityType.FOLLOW): + if actor.id != ID: + # It's a Follow from the Inbox + cache_actor_with_inbox = True + else: + # It's a new following, cache the "object" (which is the actor we follow) + DB.activities.update_one( + {"remote_id": iri}, + { + "$set": { + "meta.object": activitypub._actor_to_meta( + activity.get_object() + ) + } + }, + ) + + # Cache the actor info + DB.activities.update_one( + {"remote_id": iri}, + { + "$set": { + "meta.actor": activitypub._actor_to_meta( + actor, cache_actor_with_inbox + ) + } + }, + ) + + app.logger.info(f"actor cached for {iri}") + if also_cache_attachments and activity.has_type(ap.ActivityType.CREATE): + Tasks.cache_attachments(iri) + + except (ActivityGoneError, ActivityNotFoundError): + DB.activities.update_one({"remote_id": iri}, {"$set": {"meta.deleted": True}}) + app.logger.exception(f"flagging activity {iri} as deleted, no actor caching") + except Exception: + app.logger.exception(f"failed to cache actor for {iri}") + abort(500) + + +@app.route("/task/process_new_activity", methods=["POST"]) # noqa:c901 +def task_process_new_activity(): + """Process an activity received in the inbox""" + task = p.parse(request) + app.logger.info(f"task={task!r}") + iri = task.payload + try: + activity = ap.fetch_remote_activity(iri) + app.logger.info(f"activity={activity!r}") + + # Is the activity expected? + # following = ap.get_backend().following() + should_forward = False + should_delete = False + + tag_stream = False + if activity.has_type(ap.ActivityType.ANNOUNCE): + try: + activity.get_object() + tag_stream = True + except (NotAnActivityError, BadActivityError): + app.logger.exception(f"failed to get announce object for {activity!r}") + # Most likely on OStatus notice + tag_stream = False + should_delete = True + except (ActivityGoneError, ActivityNotFoundError): + # The announced activity is deleted/gone, drop it + should_delete = True + + elif activity.has_type(ap.ActivityType.CREATE): + note = activity.get_object() + # Make the note part of the stream if it's not a reply, or if it's a local reply + if not note.inReplyTo or note.inReplyTo.startswith(ID): + tag_stream = True + + if note.inReplyTo: + try: + reply = ap.fetch_remote_activity(note.inReplyTo) + if ( + reply.id.startswith(ID) or reply.has_mention(ID) + ) and activity.is_public(): + # The reply is public "local reply", forward the reply (i.e. the original activity) to the + # original recipients + should_forward = True + except NotAnActivityError: + # Most likely a reply to an OStatus notce + should_delete = True + + # (partial) Ghost replies handling + # [X] This is the first time the server has seen this Activity. + should_forward = False + local_followers = ID + "/followers" + for field in ["to", "cc"]: + if field in activity._data: + if local_followers in activity._data[field]: + # [X] The values of to, cc, and/or audience contain a Collection owned by the server. + should_forward = True + + # [X] The values of inReplyTo, object, target and/or tag are objects owned by the server + if not (note.inReplyTo and note.inReplyTo.startswith(ID)): + should_forward = False + + elif activity.has_type(ap.ActivityType.DELETE): + note = DB.activities.find_one( + {"activity.object.id": activity.get_object().id} + ) + if note and note["meta"].get("forwarded", False): + # If the activity was originally forwarded, forward the delete too + should_forward = True + + elif activity.has_type(ap.ActivityType.LIKE): + if not activity.get_object_id().startswith(BASE_URL): + # We only want to keep a like if it's a like for a local activity + # (Pleroma relay the likes it received, we don't want to store them) + should_delete = True + + if should_forward: + app.logger.info(f"will forward {activity!r} to followers") + Tasks.forward_activity(activity.id) + + if should_delete: + app.logger.info(f"will soft delete {activity!r}") + + app.logger.info(f"{iri} tag_stream={tag_stream}") + DB.activities.update_one( + {"remote_id": activity.id}, + { + "$set": { + "meta.stream": tag_stream, + "meta.forwarded": should_forward, + "meta.deleted": should_delete, + } + }, + ) + + app.logger.info(f"new activity {iri} processed") + if not should_delete and not activity.has_type(ap.ActivityType.DELETE): + Tasks.cache_actor(iri) + except (ActivityGoneError, ActivityNotFoundError): + app.logger.log.exception(f"dropping activity {iri}, skip processing") + except Exception: + app.logger.exception(f"failed to process new activity {iri}") + abort(500) + + +@app.route("/task/forward_activity") +def task_forward_activity(): + task = p.parse(request) + app.logger.info(f"task={task!r}") + iri = task.payload + try: + activity = ap.fetch_remote_activity(iri) + recipients = back.followers_as_recipients() + app.logger.debug(f"Forwarding {activity!r} to {recipients}") + activity = ap.clean_activity(activity.to_dict()) + payload = json.dumps(activity) + for recp in recipients: + app.logger.debug(f"forwarding {activity!r} to {recp}") + Tasks.post_to_remote_inbox(payload, recp) + except Exception: + app.logger.exception("task failed") + abort(500) + + +@app.route("/task/post_to_remote_inbox") +def task_post_to_remote_inbox(): + task = p.parse(request) + app.logger.info(f"task={task!r}") + payload, to = task.payload["payload"], task.payload["to"] + try: + app.logger.info("payload=%s", payload) + app.logger.info("generating sig") + signed_payload = json.loads(payload) + + # Don't overwrite the signature if we're forwarding an activity + if "signature" not in signed_payload: + generate_signature(signed_payload, KEY) + + app.logger.info("to=%s", to) + resp = requests.post( + to, + data=json.dumps(signed_payload), + auth=SIG_AUTH, + headers={ + "Content-Type": HEADERS[1], + "Accept": HEADERS[1], + "User-Agent": USER_AGENT, + }, + ) + app.logger.info("resp=%s", resp) + app.logger.info("resp_body=%s", resp.text) + resp.raise_for_status() + except HTTPError as err: + app.logger.exception("request failed") + if 400 >= err.response.status_code >= 499: + app.logger.info("client error, no retry") + return "" + + abort(500) diff --git a/config.py b/config.py index b66ce71..846ae79 100644 --- a/config.py +++ b/config.py @@ -105,12 +105,17 @@ MEDIA_CACHE = MediaCache(GRIDFS, USER_AGENT) def create_indexes(): DB.activities.create_index([("remote_id", pymongo.ASCENDING)]) DB.activities.create_index([("activity.object.id", pymongo.ASCENDING)]) - DB.activities.create_index([ - ("activity.object.id", pymongo.ASCENDING), - ("meta.deleted", pymongo.ASCENDING), - ]) - DB.cache2.create_index([("path", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ("arg", pymongo.ASCENDING)]) - DB.cache2.create_index("date", expireAfterSeconds=3600*12) + DB.activities.create_index( + [("activity.object.id", pymongo.ASCENDING), ("meta.deleted", pymongo.ASCENDING)] + ) + DB.cache2.create_index( + [ + ("path", pymongo.ASCENDING), + ("type", pymongo.ASCENDING), + ("arg", pymongo.ASCENDING), + ] + ) + DB.cache2.create_index("date", expireAfterSeconds=3600 * 12) # Index for the block query DB.activities.create_index( diff --git a/docker-compose-tests.yml b/docker-compose-tests.yml index eae4264..0a9c393 100644 --- a/docker-compose-tests.yml +++ b/docker-compose-tests.yml @@ -4,9 +4,6 @@ services: image: 'microblogpub:latest' ports: - "${WEB_PORT}:5005" - links: - - mongo - - rmq volumes: - "${CONFIG_DIR}:/app/config" - "./static:/app/static" @@ -14,12 +11,10 @@ services: - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// - MICROBLOGPUB_MONGODB_HOST=mongo:27017 - MICROBLOGPUB_DEBUG=1 + - POUSSETACHES_AUTH_KEY=123 celery: # image: "instance1_web" image: 'microblogpub:latest' - links: - - mongo - - rmq command: 'celery worker -l info -A tasks' volumes: - "${CONFIG_DIR}:/app/config" @@ -35,6 +30,10 @@ services: environment: - RABBITMQ_ERLANG_COOKIE=secretrabbit - RABBITMQ_NODENAME=rabbit@my-rabbit + poussetaches: + image: "poussetaches:latest" + environment: + - POUSSETACHES_AUTH_KEY=123 networks: default: name: microblogpubfede diff --git a/docker-compose.yml b/docker-compose.yml index b7f9521..e77552f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,12 +7,14 @@ services: links: - mongo - rmq + - poussetaches volumes: - "${CONFIG_DIR}:/app/config" - "./static:/app/static" environment: - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// - MICROBLOGPUB_MONGODB_HOST=mongo:27017 + - POUSSETACHES_AUTH_KEY=123 celery: image: 'microblogpub:latest' links: @@ -36,3 +38,7 @@ services: - RABBITMQ_NODENAME=rabbit@my-rabbit volumes: - "${DATA_DIR}/rabbitmq:/var/lib/rabbitmq" + poussetaches: + image: "poussetaches:latest" + environment: + - POUSSETACHES_AUTH_KEY=123 diff --git a/poussetaches.py b/poussetaches.py new file mode 100644 index 0000000..f2532bc --- /dev/null +++ b/poussetaches.py @@ -0,0 +1,48 @@ +import base64 +import json +import os +from typing import Any +from dataclasses import dataclass +import flask +import requests + +POUSSETACHES_AUTH_KEY = os.getenv("POUSSETACHES_AUTH_KEY") + + +@dataclass +class Task: + req_id: str + tries: int + + payload: Any + + +class PousseTaches: + def __init__(self, api_url: str, base_url: str) -> None: + self.api_url = api_url + self.base_url = base_url + + def push(self, payload: Any, path: str, expected=200) -> str: + # Encode our payload + p = base64.b64encode(json.dumps(payload).encode()).decode() + + # Queue/push it + resp = requests.post( + self.api_url, + json={"url": self.base_url + path, "payload": p, "expected": expected}, + ) + resp.raise_for_status() + + return resp.headers.get("Poussetaches-Task-ID") + + def parse(self, req: flask.Request) -> Task: + if req.headers.get("Poussetaches-Auth-Key") != POUSSETACHES_AUTH_KEY: + raise ValueError("Bad auth key") + + # Parse the "envelope" + envelope = json.loads(req.data) + print(req) + print(f"envelope={envelope!r}") + payload = json.loads(base64.b64decode(envelope["payload"])) + + return Task(req_id=envelope["req_id"], tries=envelope["tries"], payload=payload) diff --git a/tasks.py b/tasks.py index 7a2f64d..f34d217 100644 --- a/tasks.py +++ b/tasks.py @@ -339,6 +339,7 @@ def invalidate_cache(activity): DB.cache2.remove() # FIXME(tsileo): check if it's a reply of a reply + @app.task(bind=True, max_retries=MAX_RETRIES) # noqa: C901 def finish_post_to_inbox(self, iri: str) -> None: try: From af46e914bb17c297bd58e61c6e6e8e836fb36e63 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 11:42:14 +0200 Subject: [PATCH 0343/1425] Fix config --- docker-compose-tests.yml | 2 ++ docker-compose.yml | 2 ++ poussetaches.py | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docker-compose-tests.yml b/docker-compose-tests.yml index 0a9c393..5d93b0e 100644 --- a/docker-compose-tests.yml +++ b/docker-compose-tests.yml @@ -32,6 +32,8 @@ services: - RABBITMQ_NODENAME=rabbit@my-rabbit poussetaches: image: "poussetaches:latest" + ports: + - '7991' environment: - POUSSETACHES_AUTH_KEY=123 networks: diff --git a/docker-compose.yml b/docker-compose.yml index e77552f..bfb08a6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,5 +40,7 @@ services: - "${DATA_DIR}/rabbitmq:/var/lib/rabbitmq" poussetaches: image: "poussetaches:latest" + ports: + - '7991' environment: - POUSSETACHES_AUTH_KEY=123 diff --git a/poussetaches.py b/poussetaches.py index f2532bc..ea989b6 100644 --- a/poussetaches.py +++ b/poussetaches.py @@ -33,7 +33,7 @@ class PousseTaches: ) resp.raise_for_status() - return resp.headers.get("Poussetaches-Task-ID") + return resp.headers["Poussetaches-Task-ID"] def parse(self, req: flask.Request) -> Task: if req.headers.get("Poussetaches-Auth-Key") != POUSSETACHES_AUTH_KEY: From 6e7bfdd5c164051c46b6374265fa629d07a6a2d8 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 15:14:57 +0200 Subject: [PATCH 0344/1425] Fix CI --- app.py | 26 +++++++++++++++++++++----- run.sh | 2 +- tests/federation_test.py | 3 ++- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/app.py b/app.py index c9ac063..34894a6 100644 --- a/app.py +++ b/app.py @@ -2252,7 +2252,7 @@ def task_t2(): @app.route("/task/fetch_og_meta", methods=["POST"]) -def task_fetch_og_metadata(): +def task_fetch_og_meta(): task = p.parse(request) app.logger.info(f"task={task!r}") iri = task.payload @@ -2287,6 +2287,8 @@ def task_fetch_og_metadata(): app.logger.exception(f"failed to fetch OG metadata for {iri}") abort(500) + return "" + @app.route("/task/cache_object", methods=["POST"]) def task_cache_object(): @@ -2405,6 +2407,8 @@ def task_finish_post_to_outbox(): app.logger.exception(f"failed to post to remote inbox for {iri}") abort(500) + return "" + @app.route("/task/finish_post_to_inbox", methods=["POST"]) # noqa: C901 def task_finish_post_to_inbox(): @@ -2447,6 +2451,8 @@ def task_finish_post_to_inbox(): app.logger.exception(f"failed to cache attachments for {iri}") abort(500) + return "" + def post_to_outbox(activity: ap.BaseActivity) -> str: if activity.has_type(ap.CREATE_TYPES): @@ -2544,9 +2550,11 @@ def task_cache_attachments(): app.logger.exception(f"failed to cache attachments for {iri}") abort(500) + return "" + @app.route("/task/cache_actor", methods=["POST"]) -def task_cache_actor(): +def task_cache_actor() -> str: task = p.parse(request) app.logger.info(f"task={task!r}") iri, also_cache_attachments = ( @@ -2558,7 +2566,7 @@ def task_cache_actor(): app.logger.info(f"activity={activity!r}") if activity.has_type(ap.ActivityType.CREATE): - Tasks.fetch_og_metadata(iri) + Tasks.fetch_og_meta(iri) if activity.has_type([ap.ActivityType.LIKE, ap.ActivityType.ANNOUNCE]): Tasks.cache_object(iri) @@ -2606,6 +2614,8 @@ def task_cache_actor(): app.logger.exception(f"failed to cache actor for {iri}") abort(500) + return "" + @app.route("/task/process_new_activity", methods=["POST"]) # noqa:c901 def task_process_new_activity(): @@ -2711,8 +2721,10 @@ def task_process_new_activity(): app.logger.exception(f"failed to process new activity {iri}") abort(500) + return "" -@app.route("/task/forward_activity") + +@app.route("/task/forward_activity", methods=["POST"]) def task_forward_activity(): task = p.parse(request) app.logger.info(f"task={task!r}") @@ -2730,8 +2742,10 @@ def task_forward_activity(): app.logger.exception("task failed") abort(500) + return "" -@app.route("/task/post_to_remote_inbox") + +@app.route("/task/post_to_remote_inbox", methods=["POST"]) def task_post_to_remote_inbox(): task = p.parse(request) app.logger.info(f"task={task!r}") @@ -2766,3 +2780,5 @@ def task_post_to_remote_inbox(): return "" abort(500) + + return "" diff --git a/run.sh b/run.sh index a29f5fc..a7eef03 100755 --- a/run.sh +++ b/run.sh @@ -1,3 +1,3 @@ #!/bin/bash python -c "import config; config.create_indexes()" -gunicorn -t 300 -w 2 -b 0.0.0.0:5005 --log-level debug app:app +gunicorn -t 300 -w 5 -b 0.0.0.0:5005 --log-level debug app:app diff --git a/tests/federation_test.py b/tests/federation_test.py index 20569bb..bdaf58d 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -19,7 +19,7 @@ class Instance(object): def __init__(self, name, host_url, docker_url=None): self.host_url = host_url self.docker_url = docker_url or host_url - self._create_delay = 12 + self._create_delay = 60 with open( os.path.join( os.path.dirname(os.path.abspath(__file__)), @@ -50,6 +50,7 @@ class Instance(object): def debug(self): """Returns the debug infos (number of items in the inbox/outbox.""" + time.sleep(self._create_delay) resp = requests.get( f"{self.host_url}/api/debug", headers={**self._auth_headers, "Accept": "application/json"}, From 84997b564f97948efb95883890aa4c5705e923d9 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 20:03:49 +0200 Subject: [PATCH 0345/1425] Fix tests --- app.py | 3 ++- docker-compose-tests.yml | 3 ++- poussetaches.py | 2 +- tests/federation_test.py | 3 +-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app.py b/app.py index 34894a6..4e3ef0b 100644 --- a/app.py +++ b/app.py @@ -88,7 +88,8 @@ from utils.media import Kind from poussetaches import PousseTaches -p = PousseTaches("http://poussetaches:7991", "http://web:5005") +phost = "http://" + os.getenv("COMPOSE_PROJECT_NAME", "") +p = PousseTaches(f"{phost}_poussetaches_1:7991", f"{phost}_web_1:5005") back = activitypub.MicroblogPubBackend() diff --git a/docker-compose-tests.yml b/docker-compose-tests.yml index 5d93b0e..fc94f79 100644 --- a/docker-compose-tests.yml +++ b/docker-compose-tests.yml @@ -9,8 +9,9 @@ services: - "./static:/app/static" environment: - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// - - MICROBLOGPUB_MONGODB_HOST=mongo:27017 + - MICROBLOGPUB_MONGODB_HOST=${COMPOSE_PROJECT_NAME}_mongo_1:27017 - MICROBLOGPUB_DEBUG=1 + - COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME} - POUSSETACHES_AUTH_KEY=123 celery: # image: "instance1_web" diff --git a/poussetaches.py b/poussetaches.py index ea989b6..6bf9180 100644 --- a/poussetaches.py +++ b/poussetaches.py @@ -45,4 +45,4 @@ class PousseTaches: print(f"envelope={envelope!r}") payload = json.loads(base64.b64decode(envelope["payload"])) - return Task(req_id=envelope["req_id"], tries=envelope["tries"], payload=payload) + return Task(req_id=envelope["req_id"], tries=envelope["tries"], payload=payload) # type: ignore diff --git a/tests/federation_test.py b/tests/federation_test.py index bdaf58d..959280f 100644 --- a/tests/federation_test.py +++ b/tests/federation_test.py @@ -19,7 +19,7 @@ class Instance(object): def __init__(self, name, host_url, docker_url=None): self.host_url = host_url self.docker_url = docker_url or host_url - self._create_delay = 60 + self._create_delay = 15 with open( os.path.join( os.path.dirname(os.path.abspath(__file__)), @@ -50,7 +50,6 @@ class Instance(object): def debug(self): """Returns the debug infos (number of items in the inbox/outbox.""" - time.sleep(self._create_delay) resp = requests.get( f"{self.host_url}/api/debug", headers={**self._auth_headers, "Accept": "application/json"}, From f90a270c9c41e158e33e47beb3218b2e9668a876 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 20:06:18 +0200 Subject: [PATCH 0346/1425] Fix CI --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index db184e9..f9fc688 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,9 +24,9 @@ script: - docker build . -t microblogpub:latest - docker-compose up -d - docker-compose ps - - WEB_PORT=5006 CONFIG_DIR=./tests/fixtures/instance1/config docker-compose -p instance1 -f docker-compose-tests.yml up -d + - WEB_PORT=5006 COMPOSE_PROJECT_NAME=instance1 CONFIG_DIR=./tests/fixtures/instance1/config docker-compose -p instance1 -f docker-compose-tests.yml up -d - docker-compose -p instance1 -f docker-compose-tests.yml ps - - WEB_PORT=5007 CONFIG_DIR=./tests/fixtures/instance2/config docker-compose -p instance2 -f docker-compose-tests.yml up -d + - WEB_PORT=5007 COMPOSE_PROJECT_NAME=instance2 CONFIG_DIR=./tests/fixtures/instance2/config docker-compose -p instance2 -f docker-compose-tests.yml up -d - docker-compose -p instance2 -f docker-compose-tests.yml ps - sleep 5 # Integration tests first From ec64d24449ea265c06fe5517222d4fe268ab6a5d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 20:08:12 +0200 Subject: [PATCH 0347/1425] Re-enable Celery for the migration --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 4e3ef0b..46d3753 100644 --- a/app.py +++ b/app.py @@ -62,7 +62,7 @@ from werkzeug.utils import secure_filename import activitypub import config -# import tasks +import tasks from activitypub import Box from activitypub import embed_collection from config import USER_AGENT From eb92169de9927fb0373d9154b041a857cebcde74 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 21:36:56 +0200 Subject: [PATCH 0348/1425] More poussetaches integrations --- app.py | 102 +++++++++++++++++---------------------- docker-compose-tests.yml | 1 - docker-compose.yml | 8 +-- poussetaches.py | 61 +++++++++++++++++++++++ 4 files changed, 110 insertions(+), 62 deletions(-) diff --git a/app.py b/app.py index 46d3753..e320d83 100644 --- a/app.py +++ b/app.py @@ -62,7 +62,7 @@ from werkzeug.utils import secure_filename import activitypub import config -import tasks +import tasks # noqa: here just for the migration # FIXME(tsileo): remove me from activitypub import Box from activitypub import embed_collection from config import USER_AGENT @@ -2210,6 +2210,9 @@ def token_endpoint(): ) +################# +# Feeds + @app.route("/feed.json") def json_feed(): return Response( @@ -2234,22 +2237,48 @@ def rss_feed(): ) -@app.route("/task/t1") -def task_t1(): - p.push( - "https://mastodon.cloud/@iulius/101852467780804071/activity", - "/task/cache_object", - ) - return "ok" +########### +# Tasks +class Tasks: + @staticmethod + def cache_object(iri: str) -> None: + p.push(iri, "/task/cache_object") -@app.route("/task/t2", methods=["POST"]) -def task_t2(): - print(request) - print(request.headers) - task = p.parse(request) - print(task) - return "yay" + @staticmethod + def cache_actor(iri: str, also_cache_attachments: bool = True) -> None: + p.push( + {"iri": iri, "also_cache_attachments": also_cache_attachments}, + "/task/cache_actor", + ) + + @staticmethod + def post_to_remote_inbox(payload: str, recp: str) -> None: + p.push({"payload": payload, "to": recp}, "/task/post_to_remote_inbox") + + @staticmethod + def forward_activity(iri: str) -> None: + p.push(iri, "/task/forward_activity") + + @staticmethod + def fetch_og_meta(iri: str) -> None: + p.push(iri, "/task/fetch_og_meta") + + @staticmethod + def process_new_activity(iri: str) -> None: + p.push(iri, "/task/process_new_activity") + + @staticmethod + def cache_attachments(iri: str) -> None: + p.push(iri, "/task/cache_attachments") + + @staticmethod + def finish_post_to_inbox(iri: str) -> None: + p.push(iri, "/task/finish_post_to_inbox") + + @staticmethod + def finish_post_to_outbox(iri: str) -> None: + p.push(iri, "/task/finish_post_to_outbox") @app.route("/task/fetch_og_meta", methods=["POST"]) @@ -2321,48 +2350,6 @@ def task_cache_object(): abort(500) return "" - -class Tasks: - @staticmethod - def cache_object(iri: str) -> None: - p.push(iri, "/task/cache_object") - - @staticmethod - def cache_actor(iri: str, also_cache_attachments: bool = True) -> None: - p.push( - {"iri": iri, "also_cache_attachments": also_cache_attachments}, - "/task/cache_actor", - ) - - @staticmethod - def post_to_remote_inbox(payload: str, recp: str) -> None: - p.push({"payload": payload, "to": recp}, "/task/post_to_remote_inbox") - - @staticmethod - def forward_activity(iri: str) -> None: - p.push(iri, "/task/forward_activity") - - @staticmethod - def fetch_og_meta(iri: str) -> None: - p.push(iri, "/task/fetch_og_meta") - - @staticmethod - def process_new_activity(iri: str) -> None: - p.push(iri, "/task/process_new_activity") - - @staticmethod - def cache_attachments(iri: str) -> None: - p.push(iri, "/task/cache_attachments") - - @staticmethod - def finish_post_to_inbox(iri: str) -> None: - p.push(iri, "/task/finish_post_to_inbox") - - @staticmethod - def finish_post_to_outbox(iri: str) -> None: - p.push(iri, "/task/finish_post_to_outbox") - - @app.route("/task/finish_post_to_outbox", methods=["POST"]) # noqa:C901 def task_finish_post_to_outbox(): task = p.parse(request) @@ -2748,6 +2735,7 @@ def task_forward_activity(): @app.route("/task/post_to_remote_inbox", methods=["POST"]) def task_post_to_remote_inbox(): + """Post an activity to a remote inbox.""" task = p.parse(request) app.logger.info(f"task={task!r}") payload, to = task.payload["payload"], task.payload["to"] diff --git a/docker-compose-tests.yml b/docker-compose-tests.yml index fc94f79..3360bdc 100644 --- a/docker-compose-tests.yml +++ b/docker-compose-tests.yml @@ -33,7 +33,6 @@ services: - RABBITMQ_NODENAME=rabbit@my-rabbit poussetaches: image: "poussetaches:latest" - ports: - '7991' environment: - POUSSETACHES_AUTH_KEY=123 diff --git a/docker-compose.yml b/docker-compose.yml index bfb08a6..94c49ef 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: environment: - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// - MICROBLOGPUB_MONGODB_HOST=mongo:27017 - - POUSSETACHES_AUTH_KEY=123 + - POUSSETACHES_AUTH_KEY=${POUSSETACHES_AUTH_KEY} celery: image: 'microblogpub:latest' links: @@ -40,7 +40,7 @@ services: - "${DATA_DIR}/rabbitmq:/var/lib/rabbitmq" poussetaches: image: "poussetaches:latest" - ports: - - '7991' + volumes: + - "${DATA_DIR}/poussetaches:/app/poussetaches/poussetaches_data" environment: - - POUSSETACHES_AUTH_KEY=123 + - POUSSETACHES_AUTH_KEY=${POUSSETACHES_AUTH_KEY} diff --git a/poussetaches.py b/poussetaches.py index 6bf9180..cabf5ca 100644 --- a/poussetaches.py +++ b/poussetaches.py @@ -1,10 +1,13 @@ import base64 import json import os +from typing import Dict from typing import Any +from typing import List from dataclasses import dataclass import flask import requests +from datetime import datetime POUSSETACHES_AUTH_KEY = os.getenv("POUSSETACHES_AUTH_KEY") @@ -17,6 +20,18 @@ class Task: payload: Any +@dataclass +class GetTask: + payload: Any + expected: int + task_id: str + next_run: datetime + tries: int + url: str + last_error_status_code: int + last_error_body: str + + class PousseTaches: def __init__(self, api_url: str, base_url: str) -> None: self.api_url = api_url @@ -46,3 +61,49 @@ class PousseTaches: payload = json.loads(base64.b64decode(envelope["payload"])) return Task(req_id=envelope["req_id"], tries=envelope["tries"], payload=payload) # type: ignore + + @staticmethod + def _expand_task(t: Dict[str, Any]) -> None: + try: + t["payload"] = json.loads(base64.b64decode(t["payload"])) + except json.JSONDecodeError: + t["payload"] = base64.b64decode(t["payload"]).decode() + + if t["last_error_body"]: + t["last_error_body"] = base64.b64decode(t["last_error_body"]).decode() + + t["next_run"] = datetime.fromtimestamp(float(t["next_run"] / 1e9)) + if t["last_run"]: + t["last_run"] = datetime.fromtimestamp(float(t["last__run"] / 1e9)) + else: + del t["last_run"] + + def _get(self, where: str) -> List[GetTask]: + out = [] + + resp = requests.get(self.api_url + f"/{where}") + resp.raise_for_status() + dat = resp.json() + for t in dat["tasks"]: + self._expand_task(t) + out.append(GetTask( + task_id=t["id"], + payload=t["payload"], + expected=t["expected"], + tries=t["tries"], + url=t["url"], + last_error_status_code=t["last_error_status_code"], + last_error_body=t["last_error_body"], + next_run=t["next_run"], + )) + + return out + + def get_success(self) -> List[GetTask]: + return self._get("success") + + def get_waiting(self) -> List[GetTask]: + return self._get("waiting") + + def get_dead(self) -> List[GetTask]: + return self._get("dead") From d2b8063efe36b7a4e9a1fe1902e2969502b4a8a9 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 21:42:44 +0200 Subject: [PATCH 0349/1425] Add data dir for tasks --- data/poussetaches/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 data/poussetaches/.gitignore diff --git a/data/poussetaches/.gitignore b/data/poussetaches/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/data/poussetaches/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore From 07fdc123f354f91ef1c1d55ed8cad4cdf35956c8 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 22:11:32 +0200 Subject: [PATCH 0350/1425] Fix docker compose config --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 94c49ef..23ba974 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,6 +41,6 @@ services: poussetaches: image: "poussetaches:latest" volumes: - - "${DATA_DIR}/poussetaches:/app/poussetaches/poussetaches_data" + - "${DATA_DIR}/poussetaches:/app/poussetaches_data" environment: - POUSSETACHES_AUTH_KEY=${POUSSETACHES_AUTH_KEY} From 60f3f786313ef9ce1d1f7211c443d04ad48233b0 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 22:16:10 +0200 Subject: [PATCH 0351/1425] Fix docker compose config --- docker-compose-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose-tests.yml b/docker-compose-tests.yml index 3360bdc..4d9443d 100644 --- a/docker-compose-tests.yml +++ b/docker-compose-tests.yml @@ -33,7 +33,6 @@ services: - RABBITMQ_NODENAME=rabbit@my-rabbit poussetaches: image: "poussetaches:latest" - - '7991' environment: - POUSSETACHES_AUTH_KEY=123 networks: From 2bc1124a4f94e297f7dc63239c0710e24d5fea6a Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Apr 2019 22:41:25 +0200 Subject: [PATCH 0352/1425] Fix compose config --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 23ba974..634987e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,7 @@ services: - MICROBLOGPUB_AMQP_BROKER=pyamqp://guest@rmq// - MICROBLOGPUB_MONGODB_HOST=mongo:27017 - POUSSETACHES_AUTH_KEY=${POUSSETACHES_AUTH_KEY} + - COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME} celery: image: 'microblogpub:latest' links: From 0e41ae50d53c0cf3bc53a0b281d7aed1dc3c3273 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 7 Apr 2019 11:00:24 +0200 Subject: [PATCH 0353/1425] Fix empty attchment issue --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index e320d83..836387c 100644 --- a/app.py +++ b/app.py @@ -1821,7 +1821,7 @@ def api_new_note(): inReplyTo=reply.id if reply else None, ) - if "file" in request.files: + if "file" in request.files and request.files["file"].filename: file = request.files["file"] rfilename = secure_filename(file.filename) with BytesIO() as buf: From 51bc885a8183b35ff9586099d78fc940f0f400ad Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 7 Apr 2019 12:27:48 +0200 Subject: [PATCH 0354/1425] Show poussetaches tasks in the admin --- app.py | 14 +++++- templates/admin_tasks.html | 89 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 templates/admin_tasks.html diff --git a/app.py b/app.py index 836387c..ce7a160 100644 --- a/app.py +++ b/app.py @@ -89,7 +89,7 @@ from utils.media import Kind from poussetaches import PousseTaches phost = "http://" + os.getenv("COMPOSE_PROJECT_NAME", "") -p = PousseTaches(f"{phost}_poussetaches_1:7991", f"{phost}_web_1:5005") +p = PousseTaches("http://localhost:7991", "") # f"{phost}_poussetaches_1:7991", f"{phost}_web_1:5005") back = activitypub.MicroblogPubBackend() @@ -1399,6 +1399,18 @@ def admin(): ) +@app.route("/admin/tasks", methods=["GET"]) +@login_required +def admin_tasks(): + + return render_template( + "admin_tasks.html", + success=p.get_success(), + dead=p.get_dead(), + waiting=p.get_waiting(), + ) + + @app.route("/admin/lookup", methods=["GET", "POST"]) @login_required def admin_lookup(): diff --git a/templates/admin_tasks.html b/templates/admin_tasks.html new file mode 100644 index 0000000..53b3526 --- /dev/null +++ b/templates/admin_tasks.html @@ -0,0 +1,89 @@ +{% extends "layout.html" %} +{% import 'utils.html' as utils %} +{% block title %}Tasks - {{ config.NAME }}{% endblock %} +{% block content %} +
              +{% include "header.html" %} +
              + +

              Dead

              + + + + + + + + + + + + + {% for task in dead %} + + + + + + + + {% endfor %} + +
              #URLPayloadNext runResponse
              {{ task.task_id }}{{ task.url }} ({{ task.expected }}{{ task.payload }}{{ task.next_run }}Tries #{{ task.tries }}: {{ task.last_error_body }} ({{ task.last_error_status_code }})
              + + +

              Waiting

              + + + + + + + + + + + + + {% for task in waiting %} + + + + + + + + {% endfor %} + +
              #URLPayloadNext runResponse
              {{ task.task_id }}{{ task.url }} ({{ task.expected }}{{ task.payload }}{{ task.next_run }}Tries #{{ task.tries }}: {{ task.last_error_body }} ({{ task.last_error_status_code }})
              + + +

              Success

              + + + + + + + + + + + + + {% for task in success %} + + + + + + + + {% endfor %} + +
              #URLPayloadNext runResponse
              {{ task.task_id }}{{ task.url }} ({{ task.expected }}{{ task.payload }}{{ task.next_run }}Tries #{{ task.tries }}: {{ task.last_error_body }} ({{ task.last_error_status_code }})
              + + +
              +
              +{% endblock %} From 9206f778b54524490c26b89f3f8fb5575f8d65ca Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 7 Apr 2019 12:33:26 +0200 Subject: [PATCH 0355/1425] Disable debug stuff --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index ce7a160..a8f3971 100644 --- a/app.py +++ b/app.py @@ -89,7 +89,7 @@ from utils.media import Kind from poussetaches import PousseTaches phost = "http://" + os.getenv("COMPOSE_PROJECT_NAME", "") -p = PousseTaches("http://localhost:7991", "") # f"{phost}_poussetaches_1:7991", f"{phost}_web_1:5005") +p = PousseTaches(f"{phost}_poussetaches_1:7991", f"{phost}_web_1:5005") back = activitypub.MicroblogPubBackend() From 7d2e3dd337eea06f4b999a37aa19267358e6f58a Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 7 Apr 2019 12:40:33 +0200 Subject: [PATCH 0356/1425] Fix poussetaches client --- poussetaches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/poussetaches.py b/poussetaches.py index cabf5ca..7909416 100644 --- a/poussetaches.py +++ b/poussetaches.py @@ -74,7 +74,7 @@ class PousseTaches: t["next_run"] = datetime.fromtimestamp(float(t["next_run"] / 1e9)) if t["last_run"]: - t["last_run"] = datetime.fromtimestamp(float(t["last__run"] / 1e9)) + t["last_run"] = datetime.fromtimestamp(float(t["last_run"] / 1e9)) else: del t["last_run"] From 63ebf6ecf25619522e4b78ebe9f3a558c95b9958 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 7 Apr 2019 12:41:27 +0200 Subject: [PATCH 0357/1425] Fix task --- app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index a8f3971..0c0c707 100644 --- a/app.py +++ b/app.py @@ -2716,7 +2716,8 @@ def task_process_new_activity(): if not should_delete and not activity.has_type(ap.ActivityType.DELETE): Tasks.cache_actor(iri) except (ActivityGoneError, ActivityNotFoundError): - app.logger.log.exception(f"dropping activity {iri}, skip processing") + app.logger.exception(f"dropping activity {iri}, skip processing") + return "" except Exception: app.logger.exception(f"failed to process new activity {iri}") abort(500) From 87a1144f8834b329b21b8bb5c1e9944b9208177e Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 7 Apr 2019 12:53:16 +0200 Subject: [PATCH 0358/1425] Tweak the admin --- app.py | 2 -- templates/admin_tasks.html | 26 -------------------------- 2 files changed, 28 deletions(-) diff --git a/app.py b/app.py index 0c0c707..f7a2271 100644 --- a/app.py +++ b/app.py @@ -1402,10 +1402,8 @@ def admin(): @app.route("/admin/tasks", methods=["GET"]) @login_required def admin_tasks(): - return render_template( "admin_tasks.html", - success=p.get_success(), dead=p.get_dead(), waiting=p.get_waiting(), ) diff --git a/templates/admin_tasks.html b/templates/admin_tasks.html index 53b3526..38e5cbf 100644 --- a/templates/admin_tasks.html +++ b/templates/admin_tasks.html @@ -58,32 +58,6 @@ -

              Success

              - - - - - - - - - - - - - {% for task in success %} - - - - - - - - {% endfor %} - -
              #URLPayloadNext runResponse
              {{ task.task_id }}{{ task.url }} ({{ task.expected }}{{ task.payload }}{{ task.next_run }}Tries #{{ task.tries }}: {{ task.last_error_body }} ({{ task.last_error_status_code }})
              - -
              {% endblock %} From 41876c1d9c85d770f42997c23b4a01467d5d5e2a Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 7 Apr 2019 14:37:05 +0200 Subject: [PATCH 0359/1425] Improve task errors --- app.py | 57 ++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/app.py b/app.py index f7a2271..077b5b3 100644 --- a/app.py +++ b/app.py @@ -505,6 +505,22 @@ def handle_activitypub_error(error): return response +class TaskError(Exception): + """Raised to log the error for poussetaches.""" + def __init__(self): + self.message = traceback.format_exc() + + +@app.errorhandler(TaskError) +def handle_task_error(error): + logger.error( + f"caught activitypub error {error!r}, {traceback.format_tb(error.__traceback__)}" + ) + response = flask_jsonify({"traceback": error.message}) + response.status_code = 500 + return response + + # @app.errorhandler(Exception) # def handle_other_error(error): # logger.error( @@ -2322,10 +2338,10 @@ def task_fetch_og_meta(): app.logger.exception("bad request, no retry") return "" app.logger.exception("failed to fetch OG metadata") - abort(500) - except Exception: + raise TaskError() from http_err + except Exception as err: app.logger.exception(f"failed to fetch OG metadata for {iri}") - abort(500) + raise TaskError() from err return "" @@ -2354,10 +2370,10 @@ def task_cache_object(): except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): DB.activities.update_one({"remote_id": iri}, {"$set": {"meta.deleted": True}}) app.logger.exception(f"flagging activity {iri} as deleted, no object caching") - return "" - except Exception: + except Exception as err: app.logger.exception(f"failed to cache object for {iri}") - abort(500) + raise TaskError() from err + return "" @app.route("/task/finish_post_to_outbox", methods=["POST"]) # noqa:C901 @@ -2401,9 +2417,9 @@ def task_finish_post_to_outbox(): Tasks.post_to_remote_inbox(payload, recp) except (ActivityGoneError, ActivityNotFoundError): app.logger.exception(f"no retry") - except Exception: + except Exception as err: app.logger.exception(f"failed to post to remote inbox for {iri}") - abort(500) + raise TaskError() from err return "" @@ -2445,9 +2461,9 @@ def task_finish_post_to_inbox(): app.logger.exception("failed to invalidate cache") except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): app.logger.exception(f"no retry") - except Exception: + except Exception as err: app.logger.exception(f"failed to cache attachments for {iri}") - abort(500) + raise TaskError() from err return "" @@ -2544,9 +2560,9 @@ def task_cache_attachments(): except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): app.logger.exception(f"dropping activity {iri}, no attachment caching") - except Exception: + except Exception as err: app.logger.exception(f"failed to cache attachments for {iri}") - abort(500) + raise TaskError() from err return "" @@ -2608,9 +2624,9 @@ def task_cache_actor() -> str: except (ActivityGoneError, ActivityNotFoundError): DB.activities.update_one({"remote_id": iri}, {"$set": {"meta.deleted": True}}) app.logger.exception(f"flagging activity {iri} as deleted, no actor caching") - except Exception: + except Exception as err: app.logger.exception(f"failed to cache actor for {iri}") - abort(500) + raise TaskError() from err return "" @@ -2716,9 +2732,9 @@ def task_process_new_activity(): except (ActivityGoneError, ActivityNotFoundError): app.logger.exception(f"dropping activity {iri}, skip processing") return "" - except Exception: + except Exception as err: app.logger.exception(f"failed to process new activity {iri}") - abort(500) + raise TaskError() from err return "" @@ -2737,9 +2753,9 @@ def task_forward_activity(): for recp in recipients: app.logger.debug(f"forwarding {activity!r} to {recp}") Tasks.post_to_remote_inbox(payload, recp) - except Exception: + except Exception as err: app.logger.exception("task failed") - abort(500) + raise TaskError() from err return "" @@ -2779,6 +2795,9 @@ def task_post_to_remote_inbox(): app.logger.info("client error, no retry") return "" - abort(500) + raise TaskError() from err + except Exception as err: + app.logger.exception("task failed") + raise TaskError() from err return "" From 523b8686c75deae109ff05f06649d59edfc3dc4f Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 7 Apr 2019 16:49:35 +0200 Subject: [PATCH 0360/1425] Fix regression from debug --- app.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app.py b/app.py index 077b5b3..19275a7 100644 --- a/app.py +++ b/app.py @@ -2353,11 +2353,8 @@ def task_cache_object(): iri = task.payload try: activity = ap.fetch_remote_activity(iri) - print(activity) - print(activity.__dict__) app.logger.info(f"activity={activity!r}") - obj = activity - # obj = activity.get_object() + obj = activity.get_object() DB.activities.update_one( {"remote_id": activity.id}, { From 363dbf4b6ac473f37e0660ec9905e8c8b7f4625d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 7 Apr 2019 21:24:52 +0200 Subject: [PATCH 0361/1425] Start working on a clenaup task for old activities --- app.py | 88 ++++++++++++++++++++++++++++++++++++++++++++++--- poussetaches.py | 26 ++++++++------- 2 files changed, 99 insertions(+), 15 deletions(-) diff --git a/app.py b/app.py index 19275a7..507c767 100644 --- a/app.py +++ b/app.py @@ -6,6 +6,7 @@ import os import traceback import urllib from datetime import datetime +from datetime import timedelta from datetime import timezone from functools import wraps from io import BytesIO @@ -507,6 +508,7 @@ def handle_activitypub_error(error): class TaskError(Exception): """Raised to log the error for poussetaches.""" + def __init__(self): self.message = traceback.format_exc() @@ -1415,13 +1417,72 @@ def admin(): ) +@app.route("/admin/cleanup", methods=["GET"]) +@login_required +def admin_cleanup(): + d = (datetime.utcnow() - timedelta(days=45)).strftime("%Y-%m-%d") + + # Announce and Like cleanup + for ap_type in [ActivityType.ANNOUNCE, ActivityType.LIKE]: + # Migrate old (before meta.keep activities on the fly) + DB.activities.update_many( + { + "box": Box.INBOX.value, + "type": ap_type.value, + "meta.keep": {"$exists": False}, + "activity.object": {"$regex": f"^{BASE_URL}"}, + }, + {"$set": {"meta.keep": True}}, + ) + + DB.activities.update_many( + { + "box": Box.INBOX.value, + "type": ap_type.value, + "meta.keep": {"$exists": False}, + "activity.object.id": {"$regex": f"^{BASE_URL}"}, + }, + {"$set": {"meta.keep": True}}, + ) + + DB.activities.update_many( + { + "box": Box.INBOX.value, + "type": ap_type.value, + "meta.keep": {"$exists": False}, + }, + {"$set": {"meta.keep": False}}, + ) + # End of the migration + + # Delete old activities + DB.activities.delete_many( + { + "box": Box.INBOX.value, + "type": ap_type.value, + "meta.keep": False, + "activity.published": {"$lt": d}, + } + ) + + # And delete the soft-deleted one + DB.activities.delete_many( + { + "box": Box.INBOX.value, + "type": ap_type.value, + "meta.keep": False, + "meta.deleted": True, + } + ) + + return "OK" + + @app.route("/admin/tasks", methods=["GET"]) @login_required def admin_tasks(): return render_template( - "admin_tasks.html", - dead=p.get_dead(), - waiting=p.get_waiting(), + "admin_tasks.html", dead=p.get_dead(), waiting=p.get_waiting() ) @@ -2239,6 +2300,7 @@ def token_endpoint(): ################# # Feeds + @app.route("/feed.json") def json_feed(): return Response( @@ -2266,6 +2328,7 @@ def rss_feed(): ########### # Tasks + class Tasks: @staticmethod def cache_object(iri: str) -> None: @@ -2373,6 +2436,7 @@ def task_cache_object(): return "" + @app.route("/task/finish_post_to_outbox", methods=["POST"]) # noqa:C901 def task_finish_post_to_outbox(): task = p.parse(request) @@ -2642,12 +2706,15 @@ def task_process_new_activity(): # following = ap.get_backend().following() should_forward = False should_delete = False + should_keep = False tag_stream = False if activity.has_type(ap.ActivityType.ANNOUNCE): try: activity.get_object() tag_stream = True + if activity.get_object_id().startswith(BASE_URL): + should_keep = True except (NotAnActivityError, BadActivityError): app.logger.exception(f"failed to get announce object for {activity!r}") # Most likely on OStatus notice @@ -2657,12 +2724,21 @@ def task_process_new_activity(): # The announced activity is deleted/gone, drop it should_delete = True + elif activity.has_type(ap.ActivityType.FOLLOW): + # FIXME(tsileo): ensure it's a follow where the server is the object + should_keep = True + elif activity.has_type(ap.ActivityType.CREATE): note = activity.get_object() # Make the note part of the stream if it's not a reply, or if it's a local reply if not note.inReplyTo or note.inReplyTo.startswith(ID): tag_stream = True + if (note.inReplyTo and note.inReplyTo.startswith(ID)) or note.has_mention( + ID + ): + should_keep = True + if note.inReplyTo: try: reply = ap.fetch_remote_activity(note.inReplyTo) @@ -2672,6 +2748,7 @@ def task_process_new_activity(): # The reply is public "local reply", forward the reply (i.e. the original activity) to the # original recipients should_forward = True + should_keep = True except NotAnActivityError: # Most likely a reply to an OStatus notce should_delete = True @@ -2699,7 +2776,9 @@ def task_process_new_activity(): should_forward = True elif activity.has_type(ap.ActivityType.LIKE): - if not activity.get_object_id().startswith(BASE_URL): + if activity.get_object_id().startswith(BASE_URL): + should_keep = True + else: # We only want to keep a like if it's a like for a local activity # (Pleroma relay the likes it received, we don't want to store them) should_delete = True @@ -2716,6 +2795,7 @@ def task_process_new_activity(): {"remote_id": activity.id}, { "$set": { + "meta.keep": should_keep, "meta.stream": tag_stream, "meta.forwarded": should_forward, "meta.deleted": should_delete, diff --git a/poussetaches.py b/poussetaches.py index 7909416..28314f3 100644 --- a/poussetaches.py +++ b/poussetaches.py @@ -60,7 +60,9 @@ class PousseTaches: print(f"envelope={envelope!r}") payload = json.loads(base64.b64decode(envelope["payload"])) - return Task(req_id=envelope["req_id"], tries=envelope["tries"], payload=payload) # type: ignore + return Task( + req_id=envelope["req_id"], tries=envelope["tries"], payload=payload + ) # type: ignore @staticmethod def _expand_task(t: Dict[str, Any]) -> None: @@ -86,16 +88,18 @@ class PousseTaches: dat = resp.json() for t in dat["tasks"]: self._expand_task(t) - out.append(GetTask( - task_id=t["id"], - payload=t["payload"], - expected=t["expected"], - tries=t["tries"], - url=t["url"], - last_error_status_code=t["last_error_status_code"], - last_error_body=t["last_error_body"], - next_run=t["next_run"], - )) + out.append( + GetTask( + task_id=t["id"], + payload=t["payload"], + expected=t["expected"], + tries=t["tries"], + url=t["url"], + last_error_status_code=t["last_error_status_code"], + last_error_body=t["last_error_body"], + next_run=t["next_run"], + ) + ) return out From 27622813ecc2968f4e84a8a7545fd666a9f6a508 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 8 Apr 2019 16:41:09 +0200 Subject: [PATCH 0362/1425] More work for cleaning up old activities --- app.py | 262 +++++++++++++++++-------------------- poussetaches.py | 16 ++- templates/admin_tasks.html | 29 +++- templates/login.html | 2 +- utils/media.py | 60 +++++++++ 5 files changed, 222 insertions(+), 147 deletions(-) diff --git a/app.py b/app.py index 507c767..7d00994 100644 --- a/app.py +++ b/app.py @@ -599,23 +599,28 @@ def admin_login(): if request.method == "POST": csrf.protect() pwd = request.form.get("pass") - if pwd and verify_pass(pwd): - if devices: - resp = json.loads(request.form.get("resp")) - print(resp) - try: - u2f.complete_authentication(session["challenge"], resp) - except ValueError as exc: - print("failed", exc) - abort(401) - return - finally: - session["challenge"] = None + if devices: + resp = json.loads(request.form.get("resp")) + try: + u2f.complete_authentication(session["challenge"], resp) + except ValueError as exc: + print("failed", exc) + abort(403) + return + finally: + session["challenge"] = None session["logged_in"] = True return redirect( request.args.get("redirect") or url_for("admin_notifications") ) + elif pwd and verify_pass(pwd): + session["logged_in"] = True + return redirect( + request.args.get("redirect") or url_for("admin_notifications") + ) + elif pwd: + abort(403) else: abort(401) @@ -681,7 +686,8 @@ def u2f_register(): device, device_cert = u2f.complete_registration(session["challenge"], resp) session["challenge"] = None DB.u2f.insert_one({"device": device, "cert": device_cert}) - return "" + session["logged_in"] = False + return redirect("/login") ####### @@ -693,133 +699,6 @@ def drop_cache(): return "Done" -@app.route("/migration1_step1") -@login_required -def tmp_migrate(): - for activity in DB.outbox.find(): - activity["box"] = Box.OUTBOX.value - DB.activities.insert_one(activity) - for activity in DB.inbox.find(): - activity["box"] = Box.INBOX.value - DB.activities.insert_one(activity) - for activity in DB.replies.find(): - activity["box"] = Box.REPLIES.value - DB.activities.insert_one(activity) - return "Done" - - -@app.route("/migration1_step2") -@login_required -def tmp_migrate2(): - # Remove buggy OStatus announce - DB.activities.remove( - {"activity.object": {"$regex": f"^tag:"}, "type": ActivityType.ANNOUNCE.value} - ) - # Cache the object - for activity in DB.activities.find(): - if ( - activity["box"] == Box.OUTBOX.value - and activity["type"] == ActivityType.LIKE.value - ): - like = ap.parse_activity(activity["activity"]) - obj = like.get_object() - DB.activities.update_one( - {"remote_id": like.id}, - {"$set": {"meta.object": obj.to_dict(embed=True)}}, - ) - elif activity["type"] == ActivityType.ANNOUNCE.value: - announce = ap.parse_activity(activity["activity"]) - obj = announce.get_object() - DB.activities.update_one( - {"remote_id": announce.id}, - {"$set": {"meta.object": obj.to_dict(embed=True)}}, - ) - return "Done" - - -@app.route("/migration2") -@login_required -def tmp_migrate3(): - for activity in DB.activities.find(): - try: - activity = ap.parse_activity(activity["activity"]) - actor = activity.get_actor() - if actor.icon: - MEDIA_CACHE.cache(actor.icon["url"], Kind.ACTOR_ICON) - if activity.type == ActivityType.CREATE.value: - for attachment in activity.get_object()._data.get("attachment", []): - MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT) - except Exception: - app.logger.exception("failed") - return "Done" - - -@app.route("/migration3") -@login_required -def tmp_migrate4(): - for activity in DB.activities.find( - {"box": Box.OUTBOX.value, "type": ActivityType.UNDO.value} - ): - try: - activity = ap.parse_activity(activity["activity"]) - if activity.get_object().type == ActivityType.FOLLOW.value: - DB.activities.update_one( - {"remote_id": activity.get_object().id}, - {"$set": {"meta.undo": True}}, - ) - print(activity.get_object().to_dict()) - except Exception: - app.logger.exception("failed") - for activity in DB.activities.find( - {"box": Box.INBOX.value, "type": ActivityType.UNDO.value} - ): - try: - activity = ap.parse_activity(activity["activity"]) - if activity.get_object().type == ActivityType.FOLLOW.value: - DB.activities.update_one( - {"remote_id": activity.get_object().id}, - {"$set": {"meta.undo": True}}, - ) - print(activity.get_object().to_dict()) - except Exception: - app.logger.exception("failed") - return "Done" - - -@app.route("/migration4") -@login_required -def tmp_migrate5(): - for activity in DB.activities.find(): - Tasks.cache_actor(activity["remote_id"], also_cache_attachments=False) - - return "Done" - - -@app.route("/migration5") -@login_required -def tmp_migrate6(): - for activity in DB.activities.find(): - # tasks.cache_actor.delay(activity["remote_id"], also_cache_attachments=False) - - try: - a = ap.parse_activity(activity["activity"]) - if a.has_type([ActivityType.LIKE, ActivityType.FOLLOW]): - DB.activities.update_one( - {"remote_id": a.id}, - { - "$set": { - "meta.object_actor": activitypub._actor_to_meta( - a.get_object().get_actor() - ) - } - }, - ) - except Exception: - app.logger.exception(f"processing {activity} failed") - - return "Done" - - def paginated_query(db, q, limit=25, sort_key="_id"): older_than = newer_than = None query_sort = -1 @@ -1422,6 +1301,8 @@ def admin(): def admin_cleanup(): d = (datetime.utcnow() - timedelta(days=45)).strftime("%Y-%m-%d") + # (We keep Follow and Accept forever) + # Announce and Like cleanup for ap_type in [ActivityType.ANNOUNCE, ActivityType.LIKE]: # Migrate old (before meta.keep activities on the fly) @@ -1475,6 +1356,97 @@ def admin_cleanup(): } ) + # Create cleanup (more complicated) + # The one that mention our actor + DB.activities.update_many( + { + "box": Box.INBOX.value, + "meta.keep": {"$exists": False}, + "activity.object.tag.href": {"$regex": f"^{BASE_URL}"}, + }, + {"$set": {"meta.keep": True}}, + ) + DB.activities.update_many( + { + "box": Box.REPLIES.value, + "meta.keep": {"$exists": False}, + "activity.tag.href": {"$regex": f"^{BASE_URL}"}, + }, + {"$set": {"meta.keep": True}}, + ) + + # The replies of the outbox + DB.activities.update_many( + {"meta.thread_root_parent": {"$regex": f"^{BASE_URL}"}}, + {"$set": {"meta.keep": True}}, + ) + # Track all the threads we participated + keep_threads = [] + for data in DB.activities.find( + { + "box": Box.OUTBOX.value, + "type": ActivityType.CREATE.value, + "meta.thread_root_parent": {"$exists": True}, + } + ): + keep_threads.append(data["meta"]["thread_root_parent"]) + + for root_parent in set(keep_threads): + DB.activities.update_many( + {"meta.thread_root_parent": root_parent}, {"$set": {"meta.keep": True}} + ) + + DB.activities.update_many( + { + "box": {"$in": [Box.REPLIES.value, Box.INBOX.value]}, + "meta.keep": {"$exists": False}, + }, + {"$set": {"meta.keep": False}}, + ) + return "OK" + + +@app.route("/admin/cleanup2", methods=["GET"]) +@login_required +def admin_cleanup2(): + d = (datetime.utcnow() - timedelta(days=45)).strftime("%Y-%m-%d") + + # Go over the old Create activities + for data in DB.activities.find( + { + "box": Box.INBOX.value, + "type": ActivityType.CREATE.value, + "meta.keep": False, + "activity.published": {"$lt": d}, + } + ): + # Delete the cached attachment/ + for grid_item in MEDIA_CACHE.fs.find({"remote_id": data["remote_id"]}): + MEDIA_CACHE.fs.delete(grid_item._id) + + # Delete the Create activities that no longer have cached attachments + DB.activities.delete_many( + { + "box": Box.INBOX.value, + "type": ActivityType.CREATE.value, + "meta.keep": False, + "activity.published": {"$lt": d}, + } + ) + + # Delete old replies we don't care about + DB.activities.delete_many( + {"box": Box.REPLIES.value, "meta.keep": False, "activity.published": {"$lt": d}} + ) + + # Remove all the attachments no tied to a remote_id (post celery migration) + for grid_item in MEDIA_CACHE.fs.find( + {"kind": {"$in": ["og", "attachment"]}, "remote_id": {"$exists": False}} + ): + MEDIA_CACHE.fs.delete(grid_item._id) + + # TODO(tsileo): iterator over "actor_icon" and look for unused one in a separate task + return "OK" @@ -1482,7 +1454,10 @@ def admin_cleanup(): @login_required def admin_tasks(): return render_template( - "admin_tasks.html", dead=p.get_dead(), waiting=p.get_waiting() + "admin_tasks.html", + dead=p.get_dead(), + waiting=p.get_waiting(), + cron=[], # cron=p.get_cron(), ) @@ -2385,7 +2360,7 @@ def task_fetch_og_meta(): for og in og_metadata: if not og.get("image"): continue - MEDIA_CACHE.cache_og_image(og["image"]) + MEDIA_CACHE.cache_og_image2(og["image"], iri) app.logger.debug(f"OG metadata {og_metadata!r}") DB.activities.update_one( @@ -2613,7 +2588,7 @@ def task_cache_attachments(): or attachment.get("type") == ap.ActivityType.IMAGE.value ): try: - MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT) + MEDIA_CACHE.cache_attachment2(attachment["url"], iri) except ValueError: app.logger.exception(f"failed to cache {attachment}") @@ -2710,6 +2685,7 @@ def task_process_new_activity(): tag_stream = False if activity.has_type(ap.ActivityType.ANNOUNCE): + # FIXME(tsileo): Ensure it's follower and store into a "dead activities" DB try: activity.get_object() tag_stream = True diff --git a/poussetaches.py b/poussetaches.py index 28314f3..e844dee 100644 --- a/poussetaches.py +++ b/poussetaches.py @@ -24,6 +24,7 @@ class Task: class GetTask: payload: Any expected: int + # schedule: str task_id: str next_run: datetime tries: int @@ -37,14 +38,21 @@ class PousseTaches: self.api_url = api_url self.base_url = base_url - def push(self, payload: Any, path: str, expected=200) -> str: + def push( + self, payload: Any, path: str, expected: int = 200, schedule: str = "" + ) -> str: # Encode our payload p = base64.b64encode(json.dumps(payload).encode()).decode() # Queue/push it resp = requests.post( self.api_url, - json={"url": self.base_url + path, "payload": p, "expected": expected}, + json={ + "url": self.base_url + path, + "payload": p, + "expected": expected, + "schedule": schedule, + }, ) resp.raise_for_status() @@ -93,6 +101,7 @@ class PousseTaches: task_id=t["id"], payload=t["payload"], expected=t["expected"], + # shedule=t["schedule"], tries=t["tries"], url=t["url"], last_error_status_code=t["last_error_status_code"], @@ -103,6 +112,9 @@ class PousseTaches: return out + def get_cron(self) -> List[GetTask]: + return self._get("cron") + def get_success(self) -> List[GetTask]: return self._get("success") diff --git a/templates/admin_tasks.html b/templates/admin_tasks.html index 38e5cbf..14f6f1c 100644 --- a/templates/admin_tasks.html +++ b/templates/admin_tasks.html @@ -6,6 +6,33 @@ {% include "header.html" %}
              +

              Cron

              + + + + + + + + + + + + + + {% for task in dead %} + + + + + + + + + {% endfor %} + +
              #URLPayloadScheduleNext runResponse
              {{ task.task_id }}{{ task.url }} ({{ task.expected }}){{ task.payload }}{{ task.schedule }}{{ task.next_run }}Tries #{{ task.tries }}: {{ task.last_error_body }} ({{ task.last_error_status_code }})
              +

              Dead

              @@ -22,7 +49,7 @@ {% for task in dead %} - + diff --git a/templates/login.html b/templates/login.html index fde73d2..0d5f368 100644 --- a/templates/login.html +++ b/templates/login.html @@ -21,7 +21,7 @@ display:inline; {% if u2f_enabled %} - + {% else %} {% endif %} diff --git a/utils/media.py b/utils/media.py index 66ad02a..06559f7 100644 --- a/utils/media.py +++ b/utils/media.py @@ -60,6 +60,25 @@ class MediaCache(object): kind=Kind.OG_IMAGE.value, ) + def cache_og_image2(self, url: str, remote_id: str) -> None: + if self.fs.find_one({"url": url, "kind": Kind.OG_IMAGE.value}): + return + i = load(url, self.user_agent) + # Save the original attachment (gzipped) + i.thumbnail((100, 100)) + with BytesIO() as buf: + with GzipFile(mode="wb", fileobj=buf) as f1: + i.save(f1, format=i.format) + buf.seek(0) + self.fs.put( + buf, + url=url, + size=100, + content_type=i.get_format_mimetype(), + kind=Kind.OG_IMAGE.value, + remote_id=remote_id, + ) + def cache_attachment(self, url: str) -> None: if self.fs.find_one({"url": url, "kind": Kind.ATTACHMENT.value}): return @@ -98,6 +117,46 @@ class MediaCache(object): ) return + def cache_attachment2(self, url: str, remote_id: str) -> None: + if self.fs.find_one({"url": url, "kind": Kind.ATTACHMENT.value}): + return + if ( + url.endswith(".png") + or url.endswith(".jpg") + or url.endswith(".jpeg") + or url.endswith(".gif") + ): + i = load(url, self.user_agent) + # Save the original attachment (gzipped) + with BytesIO() as buf: + f1 = GzipFile(mode="wb", fileobj=buf) + i.save(f1, format=i.format) + f1.close() + buf.seek(0) + self.fs.put( + buf, + url=url, + size=None, + content_type=i.get_format_mimetype(), + kind=Kind.ATTACHMENT.value, + remote_id=remote_id, + ) + # Save a thumbnail (gzipped) + i.thumbnail((720, 720)) + with BytesIO() as buf: + with GzipFile(mode="wb", fileobj=buf) as f1: + i.save(f1, format=i.format) + buf.seek(0) + self.fs.put( + buf, + url=url, + size=720, + content_type=i.get_format_mimetype(), + kind=Kind.ATTACHMENT.value, + remote_id=remote_id, + ) + return + # The attachment is not an image, download and save it anyway with requests.get( url, stream=True, headers={"User-Agent": self.user_agent} @@ -115,6 +174,7 @@ class MediaCache(object): size=None, content_type=mimetypes.guess_type(url)[0], kind=Kind.ATTACHMENT.value, + remote_id=remote_id, ) def cache_actor_icon(self, url: str) -> None: From 12faea3c29ca7bf1d0dbcca27507a25a6587fbca Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 8 Apr 2019 16:54:07 +0200 Subject: [PATCH 0363/1425] Tweak tasks --- app.py | 8 ++++++++ run.sh | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 7d00994..0ce1479 100644 --- a/app.py +++ b/app.py @@ -1434,6 +1434,14 @@ def admin_cleanup2(): } ) + return "OK" + + +@app.route("/admin/cleanup3", methods=["GET"]) +@login_required +def admin_cleanup3(): + d = (datetime.utcnow() - timedelta(days=45)).strftime("%Y-%m-%d") + # Delete old replies we don't care about DB.activities.delete_many( {"box": Box.REPLIES.value, "meta.keep": False, "activity.published": {"$lt": d}} diff --git a/run.sh b/run.sh index a7eef03..8c9465c 100755 --- a/run.sh +++ b/run.sh @@ -1,3 +1,3 @@ #!/bin/bash python -c "import config; config.create_indexes()" -gunicorn -t 300 -w 5 -b 0.0.0.0:5005 --log-level debug app:app +gunicorn -t 600 -w 5 -b 0.0.0.0:5005 --log-level debug app:app From 849a6c4ea03f7c92e7a558317bb35349bfe30053 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 8 Apr 2019 16:55:46 +0200 Subject: [PATCH 0364/1425] Tweak gridfs cleanup task --- app.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 0ce1479..0c3ac1d 100644 --- a/app.py +++ b/app.py @@ -1422,7 +1422,10 @@ def admin_cleanup2(): ): # Delete the cached attachment/ for grid_item in MEDIA_CACHE.fs.find({"remote_id": data["remote_id"]}): - MEDIA_CACHE.fs.delete(grid_item._id) + try: + MEDIA_CACHE.fs.delete(grid_item._id) + except Exception: + pass # Delete the Create activities that no longer have cached attachments DB.activities.delete_many( @@ -1451,7 +1454,10 @@ def admin_cleanup3(): for grid_item in MEDIA_CACHE.fs.find( {"kind": {"$in": ["og", "attachment"]}, "remote_id": {"$exists": False}} ): - MEDIA_CACHE.fs.delete(grid_item._id) + try: + MEDIA_CACHE.fs.delete(grid_item._id) + except Exception: + pass # TODO(tsileo): iterator over "actor_icon" and look for unused one in a separate task From 55a1c2dd8804c4160f3b30cd6d737d8b8d6cb6c0 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 8 Apr 2019 17:24:50 +0200 Subject: [PATCH 0365/1425] Setup cron for the cleanup --- app.py | 336 ++++++++++++++++++++++++------------------------ poussetaches.py | 4 +- 2 files changed, 169 insertions(+), 171 deletions(-) diff --git a/app.py b/app.py index 0c3ac1d..9afd27a 100644 --- a/app.py +++ b/app.py @@ -91,6 +91,15 @@ from poussetaches import PousseTaches phost = "http://" + os.getenv("COMPOSE_PROJECT_NAME", "") p = PousseTaches(f"{phost}_poussetaches_1:7991", f"{phost}_web_1:5005") +# Setup the cron tasks +p.push({}, "/task/cleanup_part_1", schedule="@every 12h") +p.push({}, "/task/cleanup_part_2", schedule="@every 12h") +p.push({}, "/task/cleanup_part_3", schedule="@every 12h") + +# Also trigger a cleanup now +p.push({}, "/task/cleanup_part_1") +p.push({}, "/task/cleanup_part_2") +p.push({}, "/task/cleanup_part_3") back = activitypub.MicroblogPubBackend() @@ -1296,174 +1305,6 @@ def admin(): ) -@app.route("/admin/cleanup", methods=["GET"]) -@login_required -def admin_cleanup(): - d = (datetime.utcnow() - timedelta(days=45)).strftime("%Y-%m-%d") - - # (We keep Follow and Accept forever) - - # Announce and Like cleanup - for ap_type in [ActivityType.ANNOUNCE, ActivityType.LIKE]: - # Migrate old (before meta.keep activities on the fly) - DB.activities.update_many( - { - "box": Box.INBOX.value, - "type": ap_type.value, - "meta.keep": {"$exists": False}, - "activity.object": {"$regex": f"^{BASE_URL}"}, - }, - {"$set": {"meta.keep": True}}, - ) - - DB.activities.update_many( - { - "box": Box.INBOX.value, - "type": ap_type.value, - "meta.keep": {"$exists": False}, - "activity.object.id": {"$regex": f"^{BASE_URL}"}, - }, - {"$set": {"meta.keep": True}}, - ) - - DB.activities.update_many( - { - "box": Box.INBOX.value, - "type": ap_type.value, - "meta.keep": {"$exists": False}, - }, - {"$set": {"meta.keep": False}}, - ) - # End of the migration - - # Delete old activities - DB.activities.delete_many( - { - "box": Box.INBOX.value, - "type": ap_type.value, - "meta.keep": False, - "activity.published": {"$lt": d}, - } - ) - - # And delete the soft-deleted one - DB.activities.delete_many( - { - "box": Box.INBOX.value, - "type": ap_type.value, - "meta.keep": False, - "meta.deleted": True, - } - ) - - # Create cleanup (more complicated) - # The one that mention our actor - DB.activities.update_many( - { - "box": Box.INBOX.value, - "meta.keep": {"$exists": False}, - "activity.object.tag.href": {"$regex": f"^{BASE_URL}"}, - }, - {"$set": {"meta.keep": True}}, - ) - DB.activities.update_many( - { - "box": Box.REPLIES.value, - "meta.keep": {"$exists": False}, - "activity.tag.href": {"$regex": f"^{BASE_URL}"}, - }, - {"$set": {"meta.keep": True}}, - ) - - # The replies of the outbox - DB.activities.update_many( - {"meta.thread_root_parent": {"$regex": f"^{BASE_URL}"}}, - {"$set": {"meta.keep": True}}, - ) - # Track all the threads we participated - keep_threads = [] - for data in DB.activities.find( - { - "box": Box.OUTBOX.value, - "type": ActivityType.CREATE.value, - "meta.thread_root_parent": {"$exists": True}, - } - ): - keep_threads.append(data["meta"]["thread_root_parent"]) - - for root_parent in set(keep_threads): - DB.activities.update_many( - {"meta.thread_root_parent": root_parent}, {"$set": {"meta.keep": True}} - ) - - DB.activities.update_many( - { - "box": {"$in": [Box.REPLIES.value, Box.INBOX.value]}, - "meta.keep": {"$exists": False}, - }, - {"$set": {"meta.keep": False}}, - ) - return "OK" - - -@app.route("/admin/cleanup2", methods=["GET"]) -@login_required -def admin_cleanup2(): - d = (datetime.utcnow() - timedelta(days=45)).strftime("%Y-%m-%d") - - # Go over the old Create activities - for data in DB.activities.find( - { - "box": Box.INBOX.value, - "type": ActivityType.CREATE.value, - "meta.keep": False, - "activity.published": {"$lt": d}, - } - ): - # Delete the cached attachment/ - for grid_item in MEDIA_CACHE.fs.find({"remote_id": data["remote_id"]}): - try: - MEDIA_CACHE.fs.delete(grid_item._id) - except Exception: - pass - - # Delete the Create activities that no longer have cached attachments - DB.activities.delete_many( - { - "box": Box.INBOX.value, - "type": ActivityType.CREATE.value, - "meta.keep": False, - "activity.published": {"$lt": d}, - } - ) - - return "OK" - - -@app.route("/admin/cleanup3", methods=["GET"]) -@login_required -def admin_cleanup3(): - d = (datetime.utcnow() - timedelta(days=45)).strftime("%Y-%m-%d") - - # Delete old replies we don't care about - DB.activities.delete_many( - {"box": Box.REPLIES.value, "meta.keep": False, "activity.published": {"$lt": d}} - ) - - # Remove all the attachments no tied to a remote_id (post celery migration) - for grid_item in MEDIA_CACHE.fs.find( - {"kind": {"$in": ["og", "attachment"]}, "remote_id": {"$exists": False}} - ): - try: - MEDIA_CACHE.fs.delete(grid_item._id) - except Exception: - pass - - # TODO(tsileo): iterator over "actor_icon" and look for unused one in a separate task - - return "OK" - - @app.route("/admin/tasks", methods=["GET"]) @login_required def admin_tasks(): @@ -1471,7 +1312,7 @@ def admin_tasks(): "admin_tasks.html", dead=p.get_dead(), waiting=p.get_waiting(), - cron=[], # cron=p.get_cron(), + cron=p.get_cron(), ) @@ -2868,3 +2709,160 @@ def task_post_to_remote_inbox(): raise TaskError() from err return "" + + +@app.route("/task/cleanup_part_1", methods=["POST"]) +def task_cleanup_part_1(): + task = p.parse(request) + app.logger.info(f"task={task!r}") + d = (datetime.utcnow() - timedelta(days=15)).strftime("%Y-%m-%d") + + # (We keep Follow and Accept forever) + + # Announce and Like cleanup + for ap_type in [ActivityType.ANNOUNCE, ActivityType.LIKE]: + # Migrate old (before meta.keep activities on the fly) + DB.activities.update_many( + { + "box": Box.INBOX.value, + "type": ap_type.value, + "meta.keep": {"$exists": False}, + "activity.object": {"$regex": f"^{BASE_URL}"}, + }, + {"$set": {"meta.keep": True}}, + ) + + DB.activities.update_many( + { + "box": Box.INBOX.value, + "type": ap_type.value, + "meta.keep": {"$exists": False}, + "activity.object.id": {"$regex": f"^{BASE_URL}"}, + }, + {"$set": {"meta.keep": True}}, + ) + + DB.activities.update_many( + { + "box": Box.INBOX.value, + "type": ap_type.value, + "meta.keep": {"$exists": False}, + }, + {"$set": {"meta.keep": False}}, + ) + # End of the migration + + # Delete old activities + DB.activities.delete_many( + { + "box": Box.INBOX.value, + "type": ap_type.value, + "meta.keep": False, + "activity.published": {"$lt": d}, + } + ) + + # And delete the soft-deleted one + DB.activities.delete_many( + { + "box": Box.INBOX.value, + "type": ap_type.value, + "meta.keep": False, + "meta.deleted": True, + } + ) + + # Create cleanup (more complicated) + # The one that mention our actor + DB.activities.update_many( + { + "box": Box.INBOX.value, + "meta.keep": {"$exists": False}, + "activity.object.tag.href": {"$regex": f"^{BASE_URL}"}, + }, + {"$set": {"meta.keep": True}}, + ) + DB.activities.update_many( + { + "box": Box.REPLIES.value, + "meta.keep": {"$exists": False}, + "activity.tag.href": {"$regex": f"^{BASE_URL}"}, + }, + {"$set": {"meta.keep": True}}, + ) + + # The replies of the outbox + DB.activities.update_many( + {"meta.thread_root_parent": {"$regex": f"^{BASE_URL}"}}, + {"$set": {"meta.keep": True}}, + ) + # Track all the threads we participated + keep_threads = [] + for data in DB.activities.find( + { + "box": Box.OUTBOX.value, + "type": ActivityType.CREATE.value, + "meta.thread_root_parent": {"$exists": True}, + } + ): + keep_threads.append(data["meta"]["thread_root_parent"]) + + for root_parent in set(keep_threads): + DB.activities.update_many( + {"meta.thread_root_parent": root_parent}, {"$set": {"meta.keep": True}} + ) + + DB.activities.update_many( + { + "box": {"$in": [Box.REPLIES.value, Box.INBOX.value]}, + "meta.keep": {"$exists": False}, + }, + {"$set": {"meta.keep": False}}, + ) + return "OK" + + +@app.route("/task/cleanup_part_2", methods=["POST"]) +def task_cleanup_part_2(): + task = p.parse(request) + app.logger.info(f"task={task!r}") + d = (datetime.utcnow() - timedelta(days=15)).strftime("%Y-%m-%d") + + # Go over the old Create activities + for data in DB.activities.find( + { + "box": Box.INBOX.value, + "type": ActivityType.CREATE.value, + "meta.keep": False, + "activity.published": {"$lt": d}, + } + ): + # Delete the cached attachment/ + for grid_item in MEDIA_CACHE.fs.find({"remote_id": data["remote_id"]}): + MEDIA_CACHE.fs.delete(grid_item._id) + DB.activities.delete_one({"_id": data["_id"]}) + + return "OK" + + +@app.route("/task/cleanup_part_3", methods=["POST"]) +def task_cleanup_part_3(): + task = p.parse(request) + app.logger.info(f"task={task!r}") + + d = (datetime.utcnow() - timedelta(days=15)).strftime("%Y-%m-%d") + + # Delete old replies we don't care about + DB.activities.delete_many( + {"box": Box.REPLIES.value, "meta.keep": False, "activity.published": {"$lt": d}} + ) + + # Remove all the attachments no tied to a remote_id (post celery migration) + for grid_item in MEDIA_CACHE.fs.find( + {"kind": {"$in": ["og", "attachment"]}, "remote_id": {"$exists": False}} + ): + MEDIA_CACHE.fs.delete(grid_item._id) + + # TODO(tsileo): iterator over "actor_icon" and look for unused one in a separate task + + return "OK" diff --git a/poussetaches.py b/poussetaches.py index e844dee..6f06e14 100644 --- a/poussetaches.py +++ b/poussetaches.py @@ -24,7 +24,7 @@ class Task: class GetTask: payload: Any expected: int - # schedule: str + schedule: str task_id: str next_run: datetime tries: int @@ -101,7 +101,7 @@ class PousseTaches: task_id=t["id"], payload=t["payload"], expected=t["expected"], - # shedule=t["schedule"], + shedule=t["schedule"], tries=t["tries"], url=t["url"], last_error_status_code=t["last_error_status_code"], From 643ba9e775779f3edf66b732ec22b01a2920e83f Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 8 Apr 2019 17:30:24 +0200 Subject: [PATCH 0366/1425] Bugfix --- app.py | 6 ------ poussetaches.py | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/app.py b/app.py index 9afd27a..906af26 100644 --- a/app.py +++ b/app.py @@ -96,12 +96,6 @@ p.push({}, "/task/cleanup_part_1", schedule="@every 12h") p.push({}, "/task/cleanup_part_2", schedule="@every 12h") p.push({}, "/task/cleanup_part_3", schedule="@every 12h") -# Also trigger a cleanup now -p.push({}, "/task/cleanup_part_1") -p.push({}, "/task/cleanup_part_2") -p.push({}, "/task/cleanup_part_3") - - back = activitypub.MicroblogPubBackend() ap.use_backend(back) diff --git a/poussetaches.py b/poussetaches.py index 6f06e14..2e61105 100644 --- a/poussetaches.py +++ b/poussetaches.py @@ -101,7 +101,7 @@ class PousseTaches: task_id=t["id"], payload=t["payload"], expected=t["expected"], - shedule=t["schedule"], + schedule=t["schedule"], tries=t["tries"], url=t["url"], last_error_status_code=t["last_error_status_code"], From 533cb3e771866bfb76424bc04e7776574368e3c6 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 8 Apr 2019 17:39:29 +0200 Subject: [PATCH 0367/1425] Tweak the cleanup --- app.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app.py b/app.py index 906af26..2cbbe7c 100644 --- a/app.py +++ b/app.py @@ -91,10 +91,6 @@ from poussetaches import PousseTaches phost = "http://" + os.getenv("COMPOSE_PROJECT_NAME", "") p = PousseTaches(f"{phost}_poussetaches_1:7991", f"{phost}_web_1:5005") -# Setup the cron tasks -p.push({}, "/task/cleanup_part_1", schedule="@every 12h") -p.push({}, "/task/cleanup_part_2", schedule="@every 12h") -p.push({}, "/task/cleanup_part_3", schedule="@every 12h") back = activitypub.MicroblogPubBackend() ap.use_backend(back) @@ -1268,6 +1264,11 @@ def outbox_activity_shares(item_id): @app.route("/admin", methods=["GET"]) @login_required def admin(): + # Setup the cron tasks + p.push({}, "/task/cleanup_part_1", schedule="@every 12h") + p.push({}, "/task/cleanup_part_2", schedule="@every 12h") + p.push({}, "/task/cleanup_part_3", schedule="@every 12h") + q = { "meta.deleted": False, "meta.undo": False, From 035c08735e35f7fab72173b1e52fa5adb69c56c1 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 8 Apr 2019 17:47:16 +0200 Subject: [PATCH 0368/1425] Admin fixes --- templates/admin_tasks.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/admin_tasks.html b/templates/admin_tasks.html index 14f6f1c..d0b300e 100644 --- a/templates/admin_tasks.html +++ b/templates/admin_tasks.html @@ -20,7 +20,7 @@ - {% for task in dead %} + {% for task in cron %} @@ -75,7 +75,7 @@ {% for task in waiting %} - + From 049607e7019f654ed7c061878830ad7efe982c51 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 8 Apr 2019 17:54:51 +0200 Subject: [PATCH 0369/1425] Fix login form --- templates/login.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/templates/login.html b/templates/login.html index 0d5f368..300b866 100644 --- a/templates/login.html +++ b/templates/login.html @@ -21,10 +21,9 @@ display:inline; {% if u2f_enabled %} - {% else %} - {% endif %} + From 0e2ecab78f6f6b979d67a3b27efdbb51f52a797b Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 8 Apr 2019 18:01:02 +0200 Subject: [PATCH 0370/1425] Fix login --- app.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/app.py b/app.py index 2cbbe7c..5235402 100644 --- a/app.py +++ b/app.py @@ -597,8 +597,18 @@ def admin_login(): u2f_enabled = True if devices else False if request.method == "POST": csrf.protect() + # 1. Check regular password login flow pwd = request.form.get("pass") - if devices: + if pwd: + if verify_pass(pwd): + session["logged_in"] = True + return redirect( + request.args.get("redirect") or url_for("admin_notifications") + ) + else: + abort(403) + # 2. Check for U2F payload, if any + elif devices: resp = json.loads(request.form.get("resp")) try: u2f.complete_authentication(session["challenge"], resp) @@ -613,13 +623,6 @@ def admin_login(): return redirect( request.args.get("redirect") or url_for("admin_notifications") ) - elif pwd and verify_pass(pwd): - session["logged_in"] = True - return redirect( - request.args.get("redirect") or url_for("admin_notifications") - ) - elif pwd: - abort(403) else: abort(401) From 61aff326de51533cffab39c931d7d126416f55dd Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 8 Apr 2019 18:09:33 +0200 Subject: [PATCH 0371/1425] Cleanup the cleanup tasks --- app.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/app.py b/app.py index 5235402..f74bfa6 100644 --- a/app.py +++ b/app.py @@ -1267,11 +1267,6 @@ def outbox_activity_shares(item_id): @app.route("/admin", methods=["GET"]) @login_required def admin(): - # Setup the cron tasks - p.push({}, "/task/cleanup_part_1", schedule="@every 12h") - p.push({}, "/task/cleanup_part_2", schedule="@every 12h") - p.push({}, "/task/cleanup_part_3", schedule="@every 12h") - q = { "meta.deleted": False, "meta.undo": False, @@ -1393,6 +1388,13 @@ def admin_new(): @app.route("/admin/notifications") @login_required def admin_notifications(): + # Setup the cron for deleting old activities + p.push({}, "/task/cleanup", schedule="@every 12h") + + # Trigger a cleanup if asked + if request.args.get("cleanup"): + p.push({}, "/task/cleanup") + # FIXME(tsileo): show unfollow (performed by the current actor) and liked??? mentions_query = { "type": ActivityType.CREATE.value, @@ -2709,6 +2711,15 @@ def task_post_to_remote_inbox(): return "" +@app.route("/task/cleanup", methods=["POST"]) +def task_cleanup(): + task = p.parse(request) + app.logger.info(f"task={task!r}") + p.push({}, "/task/cleanup_part_1") + p.push({}, "/task/cleanup_part_2") + p.push({}, "/task/cleanup_part_3") + + @app.route("/task/cleanup_part_1", methods=["POST"]) def task_cleanup_part_1(): task = p.parse(request) From 143b0953bee7ac93dfd85f545eb39ddd9ac249ca Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 8 Apr 2019 18:14:39 +0200 Subject: [PATCH 0372/1425] Fix task --- app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app.py b/app.py index f74bfa6..00580bc 100644 --- a/app.py +++ b/app.py @@ -2718,6 +2718,7 @@ def task_cleanup(): p.push({}, "/task/cleanup_part_1") p.push({}, "/task/cleanup_part_2") p.push({}, "/task/cleanup_part_3") + return "" @app.route("/task/cleanup_part_1", methods=["POST"]) From 8a57d0dfdaf3b291123a7413f0e028f9b7cb7540 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 8 Apr 2019 19:54:06 +0200 Subject: [PATCH 0373/1425] More cleanup --- Makefile | 3 ++ activitypub.py | 136 ++++++++++++++++++------------------------------- app.py | 27 +++++++--- tasks.py | 6 +-- 4 files changed, 76 insertions(+), 96 deletions(-) diff --git a/Makefile b/Makefile index daad832..1cd4e26 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,9 @@ reload-dev: # docker build . -t microblogpub:latest docker-compose -f docker-compose-dev.yml up -d --force-recreate +update-poussetaches: + git clone https://github.com/tsileo/poussetaches.git tmp_poussetaches && cd tmp_poussetaches && docker build . -t poussetaches:latest && cd - && rm -rf tmp_poussetaches + update: git pull docker build . -t microblogpub:latest diff --git a/activitypub.py b/activitypub.py index 9185745..20b5f4c 100644 --- a/activitypub.py +++ b/activitypub.py @@ -1,6 +1,5 @@ import logging import os -import json from datetime import datetime from enum import Enum from typing import Any @@ -79,6 +78,52 @@ class Box(Enum): REPLIES = "replies" +def save(box: Box, activity: ap.BaseActivity) -> None: + """Custom helper for saving an activity to the DB.""" + DB.activities.insert_one( + { + "box": box.value, + "activity": activity.to_dict(), + "type": _to_list(activity.type), + "remote_id": activity.id, + "meta": {"undo": False, "deleted": False}, + } + ) + + +def followers() -> List[str]: + q = { + "box": Box.INBOX.value, + "type": ap.ActivityType.FOLLOW.value, + "meta.undo": False, + } + return [doc["activity"]["actor"] for doc in DB.activities.find(q)] + + +def following() -> List[str]: + q = { + "box": Box.OUTBOX.value, + "type": ap.ActivityType.FOLLOW.value, + "meta.undo": False, + } + return [doc["activity"]["object"] for doc in DB.activities.find(q)] + + +def followers_as_recipients() -> List[str]: + q = { + "box": Box.INBOX.value, + "type": ap.ActivityType.FOLLOW.value, + "meta.undo": False, + } + recipients = [] + for doc in DB.activities.find(q): + recipients.append( + doc["meta"]["actor"]["sharedInbox"] or doc["meta"]["actor"]["inbox"] + ) + + return list(set(recipients)) + + class MicroblogPubBackend(Backend): """Implements a Little Boxes backend, backed by MongoDB.""" @@ -104,73 +149,18 @@ class MicroblogPubBackend(Backend): """URL for activity link.""" return f"{BASE_URL}/note/{obj_id}" - def save(self, box: Box, activity: ap.BaseActivity) -> None: - """Custom helper for saving an activity to the DB.""" - DB.activities.insert_one( - { - "box": box.value, - "activity": activity.to_dict(), - "type": _to_list(activity.type), - "remote_id": activity.id, - "meta": {"undo": False, "deleted": False}, - } - ) - - def followers(self) -> List[str]: - q = { - "box": Box.INBOX.value, - "type": ap.ActivityType.FOLLOW.value, - "meta.undo": False, - } - return [doc["activity"]["actor"] for doc in DB.activities.find(q)] - - def followers_as_recipients(self) -> List[str]: - q = { - "box": Box.INBOX.value, - "type": ap.ActivityType.FOLLOW.value, - "meta.undo": False, - } - recipients = [] - for doc in DB.activities.find(q): - recipients.append( - doc["meta"]["actor"]["sharedInbox"] or doc["meta"]["actor"]["inbox"] - ) - - return list(set(recipients)) - - def following(self) -> List[str]: - q = { - "box": Box.OUTBOX.value, - "type": ap.ActivityType.FOLLOW.value, - "meta.undo": False, - } - return [doc["activity"]["object"] for doc in DB.activities.find(q)] - def parse_collection( self, payload: Optional[Dict[str, Any]] = None, url: Optional[str] = None ) -> List[str]: """Resolve/fetch a `Collection`/`OrderedCollection`.""" # Resolve internal collections via MongoDB directly if url == ID + "/followers": - return self.followers() + return followers() elif url == ID + "/following": - return self.following() + return following() return super().parse_collection(payload, url) - @ensure_it_is_me - def outbox_is_blocked(self, as_actor: ap.Person, actor_id: str) -> bool: - return bool( - DB.activities.find_one( - { - "box": Box.OUTBOX.value, - "type": ap.ActivityType.BLOCK.value, - "activity.object": actor_id, - "meta.undo": False, - } - ) - ) - def _fetch_iri(self, iri: str) -> ap.ObjectType: if iri == ME["id"]: return ME @@ -229,13 +219,6 @@ class MicroblogPubBackend(Backend): return data - @ensure_it_is_me - def inbox_check_duplicate(self, as_actor: ap.Person, iri: str) -> bool: - return bool(DB.activities.find_one({"box": Box.INBOX.value, "remote_id": iri})) - - def set_post_to_remote_inbox(self, cb): - self.post_to_remote_inbox_cb = cb - @ensure_it_is_me def undo_new_follower(self, as_actor: ap.Person, follow: ap.Follow) -> None: DB.activities.update_one( @@ -471,7 +454,7 @@ class MicroblogPubBackend(Backend): ) if not creply: # It means the activity is not in the inbox, and not in the outbox, we want to save it - self.save(Box.REPLIES, reply) + save(Box.REPLIES, reply) new_threads.append(reply.id) while reply is not None: @@ -482,7 +465,7 @@ class MicroblogPubBackend(Backend): reply = ap.fetch_remote_activity(root_reply) q = {"activity.object.id": root_reply} if not DB.activities.count(q): - self.save(Box.REPLIES, reply) + save(Box.REPLIES, reply) new_threads.append(reply.id) DB.activities.update_one( @@ -493,25 +476,6 @@ class MicroblogPubBackend(Backend): {"$set": {"meta.thread_root_parent": root_reply}}, ) - def post_to_outbox(self, activity: ap.BaseActivity) -> None: - if activity.has_type(ap.CREATE_TYPES): - activity = activity.build_create() - - self.save(Box.OUTBOX, activity) - - # Assign create a random ID - obj_id = self.random_object_id() - activity.set_id(self.activity_url(obj_id), obj_id) - - recipients = activity.recipients() - logger.info(f"recipients={recipients}") - activity = ap.clean_activity(activity.to_dict()) - - payload = json.dumps(activity) - for recp in recipients: - logger.debug(f"posting to {recp}") - self.post_to_remote_inbox(self.get_actor(), payload, recp) - def gen_feed(): fg = FeedGenerator() diff --git a/app.py b/app.py index 00580bc..a2b9cfc 100644 --- a/app.py +++ b/app.py @@ -66,6 +66,8 @@ import config import tasks # noqa: here just for the migration # FIXME(tsileo): remove me from activitypub import Box from activitypub import embed_collection +from activitypub import save +from activitypub import followers_as_recipients from config import USER_AGENT from config import ADMIN_API_KEY from config import BASE_URL @@ -1643,7 +1645,7 @@ def inbox(): data["object"] ): logger.info(f"received a Delete for an actor {data!r}") - if get_backend().inbox_check_duplicate(MY_PERSON, data["id"]): + if DB.activities.find_one({"box": Box.INBOX.value, "remote_id": data["id"]}): # The activity is already in the inbox logger.info(f"received duplicate activity {data!r}, dropping it") @@ -2295,7 +2297,9 @@ def task_finish_post_to_outbox(): elif obj.has_type(ap.ActivityType.ANNOUNCE): back.outbox_undo_announce(MY_PERSON, obj) elif obj.has_type(ap.ActivityType.FOLLOW): - back.undo_new_following(MY_PERSON, obj) + DB.activities.update_one( + {"remote_id": obj.id}, {"$set": {"meta.undo": True}} + ) app.logger.info(f"recipients={recipients}") activity = ap.clean_activity(activity.to_dict()) @@ -2345,7 +2349,9 @@ def task_finish_post_to_inbox(): elif obj.has_type(ap.ActivityType.ANNOUNCE): back.inbox_undo_announce(MY_PERSON, obj) elif obj.has_type(ap.ActivityType.FOLLOW): - back.undo_new_follower(MY_PERSON, obj) + DB.activities.update_one( + {"remote_id": obj.id}, {"$set": {"meta.undo": True}} + ) try: invalidate_cache(activity) except Exception: @@ -2367,7 +2373,7 @@ def post_to_outbox(activity: ap.BaseActivity) -> str: obj_id = back.random_object_id() activity.set_id(back.activity_url(obj_id), obj_id) - back.save(Box.OUTBOX, activity) + save(Box.OUTBOX, activity) Tasks.cache_actor(activity.id) Tasks.finish_post_to_outbox(activity.id) return activity.id @@ -2376,7 +2382,14 @@ def post_to_outbox(activity: ap.BaseActivity) -> str: def post_to_inbox(activity: ap.BaseActivity) -> None: # Check for Block activity actor = activity.get_actor() - if back.outbox_is_blocked(MY_PERSON, actor.id): + if DB.activities.find_one( + { + "box": Box.OUTBOX.value, + "type": ap.ActivityType.BLOCK.value, + "activity.object": actor.id, + "meta.undo": False, + } + ): app.logger.info( f"actor {actor!r} is blocked, dropping the received activity {activity!r}" ) @@ -2386,7 +2399,7 @@ def post_to_inbox(activity: ap.BaseActivity) -> None: # The activity is already in the inbox app.logger.info(f"received duplicate activity {activity!r}, dropping it") - back.save(Box.INBOX, activity) + save(Box.INBOX, activity) Tasks.process_new_activity(activity.id) app.logger.info(f"spawning task for {activity!r}") @@ -2654,7 +2667,7 @@ def task_forward_activity(): iri = task.payload try: activity = ap.fetch_remote_activity(iri) - recipients = back.followers_as_recipients() + recipients = followers_as_recipients() app.logger.debug(f"Forwarding {activity!r} to {recipients}") activity = ap.clean_activity(activity.to_dict()) payload = json.dumps(activity) diff --git a/tasks.py b/tasks.py index f34d217..519dd48 100644 --- a/tasks.py +++ b/tasks.py @@ -312,7 +312,7 @@ def post_to_inbox(activity: ap.BaseActivity) -> None: # The activity is already in the inbox log.info(f"received duplicate activity {activity!r}, dropping it") - back.save(Box.INBOX, activity) + activitypub.save(Box.INBOX, activity) process_new_activity.delay(activity.id) log.info(f"spawning task for {activity!r}") @@ -387,7 +387,7 @@ def post_to_outbox(activity: ap.BaseActivity) -> str: obj_id = back.random_object_id() activity.set_id(back.activity_url(obj_id), obj_id) - back.save(Box.OUTBOX, activity) + activitypub.save(Box.OUTBOX, activity) cache_actor.delay(activity.id) finish_post_to_outbox.delay(activity.id) return activity.id @@ -440,7 +440,7 @@ def finish_post_to_outbox(self, iri: str) -> None: def forward_activity(self, iri: str) -> None: try: activity = ap.fetch_remote_activity(iri) - recipients = back.followers_as_recipients() + recipients = activitypub.followers_as_recipients() log.debug(f"Forwarding {activity!r} to {recipients}") activity = ap.clean_activity(activity.to_dict()) payload = json.dumps(activity) From 5811583163faf4d5a9121c75f0b959e670b2449c Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 8 Apr 2019 20:55:03 +0200 Subject: [PATCH 0374/1425] Revert "More cleanup" This reverts commit 8a57d0dfdaf3b291123a7413f0e028f9b7cb7540. --- Makefile | 3 -- activitypub.py | 136 +++++++++++++++++++++++++++++++------------------ app.py | 27 +++------- tasks.py | 6 +-- 4 files changed, 96 insertions(+), 76 deletions(-) diff --git a/Makefile b/Makefile index 1cd4e26..daad832 100644 --- a/Makefile +++ b/Makefile @@ -18,9 +18,6 @@ reload-dev: # docker build . -t microblogpub:latest docker-compose -f docker-compose-dev.yml up -d --force-recreate -update-poussetaches: - git clone https://github.com/tsileo/poussetaches.git tmp_poussetaches && cd tmp_poussetaches && docker build . -t poussetaches:latest && cd - && rm -rf tmp_poussetaches - update: git pull docker build . -t microblogpub:latest diff --git a/activitypub.py b/activitypub.py index 20b5f4c..9185745 100644 --- a/activitypub.py +++ b/activitypub.py @@ -1,5 +1,6 @@ import logging import os +import json from datetime import datetime from enum import Enum from typing import Any @@ -78,52 +79,6 @@ class Box(Enum): REPLIES = "replies" -def save(box: Box, activity: ap.BaseActivity) -> None: - """Custom helper for saving an activity to the DB.""" - DB.activities.insert_one( - { - "box": box.value, - "activity": activity.to_dict(), - "type": _to_list(activity.type), - "remote_id": activity.id, - "meta": {"undo": False, "deleted": False}, - } - ) - - -def followers() -> List[str]: - q = { - "box": Box.INBOX.value, - "type": ap.ActivityType.FOLLOW.value, - "meta.undo": False, - } - return [doc["activity"]["actor"] for doc in DB.activities.find(q)] - - -def following() -> List[str]: - q = { - "box": Box.OUTBOX.value, - "type": ap.ActivityType.FOLLOW.value, - "meta.undo": False, - } - return [doc["activity"]["object"] for doc in DB.activities.find(q)] - - -def followers_as_recipients() -> List[str]: - q = { - "box": Box.INBOX.value, - "type": ap.ActivityType.FOLLOW.value, - "meta.undo": False, - } - recipients = [] - for doc in DB.activities.find(q): - recipients.append( - doc["meta"]["actor"]["sharedInbox"] or doc["meta"]["actor"]["inbox"] - ) - - return list(set(recipients)) - - class MicroblogPubBackend(Backend): """Implements a Little Boxes backend, backed by MongoDB.""" @@ -149,18 +104,73 @@ class MicroblogPubBackend(Backend): """URL for activity link.""" return f"{BASE_URL}/note/{obj_id}" + def save(self, box: Box, activity: ap.BaseActivity) -> None: + """Custom helper for saving an activity to the DB.""" + DB.activities.insert_one( + { + "box": box.value, + "activity": activity.to_dict(), + "type": _to_list(activity.type), + "remote_id": activity.id, + "meta": {"undo": False, "deleted": False}, + } + ) + + def followers(self) -> List[str]: + q = { + "box": Box.INBOX.value, + "type": ap.ActivityType.FOLLOW.value, + "meta.undo": False, + } + return [doc["activity"]["actor"] for doc in DB.activities.find(q)] + + def followers_as_recipients(self) -> List[str]: + q = { + "box": Box.INBOX.value, + "type": ap.ActivityType.FOLLOW.value, + "meta.undo": False, + } + recipients = [] + for doc in DB.activities.find(q): + recipients.append( + doc["meta"]["actor"]["sharedInbox"] or doc["meta"]["actor"]["inbox"] + ) + + return list(set(recipients)) + + def following(self) -> List[str]: + q = { + "box": Box.OUTBOX.value, + "type": ap.ActivityType.FOLLOW.value, + "meta.undo": False, + } + return [doc["activity"]["object"] for doc in DB.activities.find(q)] + def parse_collection( self, payload: Optional[Dict[str, Any]] = None, url: Optional[str] = None ) -> List[str]: """Resolve/fetch a `Collection`/`OrderedCollection`.""" # Resolve internal collections via MongoDB directly if url == ID + "/followers": - return followers() + return self.followers() elif url == ID + "/following": - return following() + return self.following() return super().parse_collection(payload, url) + @ensure_it_is_me + def outbox_is_blocked(self, as_actor: ap.Person, actor_id: str) -> bool: + return bool( + DB.activities.find_one( + { + "box": Box.OUTBOX.value, + "type": ap.ActivityType.BLOCK.value, + "activity.object": actor_id, + "meta.undo": False, + } + ) + ) + def _fetch_iri(self, iri: str) -> ap.ObjectType: if iri == ME["id"]: return ME @@ -219,6 +229,13 @@ class MicroblogPubBackend(Backend): return data + @ensure_it_is_me + def inbox_check_duplicate(self, as_actor: ap.Person, iri: str) -> bool: + return bool(DB.activities.find_one({"box": Box.INBOX.value, "remote_id": iri})) + + def set_post_to_remote_inbox(self, cb): + self.post_to_remote_inbox_cb = cb + @ensure_it_is_me def undo_new_follower(self, as_actor: ap.Person, follow: ap.Follow) -> None: DB.activities.update_one( @@ -454,7 +471,7 @@ class MicroblogPubBackend(Backend): ) if not creply: # It means the activity is not in the inbox, and not in the outbox, we want to save it - save(Box.REPLIES, reply) + self.save(Box.REPLIES, reply) new_threads.append(reply.id) while reply is not None: @@ -465,7 +482,7 @@ class MicroblogPubBackend(Backend): reply = ap.fetch_remote_activity(root_reply) q = {"activity.object.id": root_reply} if not DB.activities.count(q): - save(Box.REPLIES, reply) + self.save(Box.REPLIES, reply) new_threads.append(reply.id) DB.activities.update_one( @@ -476,6 +493,25 @@ class MicroblogPubBackend(Backend): {"$set": {"meta.thread_root_parent": root_reply}}, ) + def post_to_outbox(self, activity: ap.BaseActivity) -> None: + if activity.has_type(ap.CREATE_TYPES): + activity = activity.build_create() + + self.save(Box.OUTBOX, activity) + + # Assign create a random ID + obj_id = self.random_object_id() + activity.set_id(self.activity_url(obj_id), obj_id) + + recipients = activity.recipients() + logger.info(f"recipients={recipients}") + activity = ap.clean_activity(activity.to_dict()) + + payload = json.dumps(activity) + for recp in recipients: + logger.debug(f"posting to {recp}") + self.post_to_remote_inbox(self.get_actor(), payload, recp) + def gen_feed(): fg = FeedGenerator() diff --git a/app.py b/app.py index a2b9cfc..00580bc 100644 --- a/app.py +++ b/app.py @@ -66,8 +66,6 @@ import config import tasks # noqa: here just for the migration # FIXME(tsileo): remove me from activitypub import Box from activitypub import embed_collection -from activitypub import save -from activitypub import followers_as_recipients from config import USER_AGENT from config import ADMIN_API_KEY from config import BASE_URL @@ -1645,7 +1643,7 @@ def inbox(): data["object"] ): logger.info(f"received a Delete for an actor {data!r}") - if DB.activities.find_one({"box": Box.INBOX.value, "remote_id": data["id"]}): + if get_backend().inbox_check_duplicate(MY_PERSON, data["id"]): # The activity is already in the inbox logger.info(f"received duplicate activity {data!r}, dropping it") @@ -2297,9 +2295,7 @@ def task_finish_post_to_outbox(): elif obj.has_type(ap.ActivityType.ANNOUNCE): back.outbox_undo_announce(MY_PERSON, obj) elif obj.has_type(ap.ActivityType.FOLLOW): - DB.activities.update_one( - {"remote_id": obj.id}, {"$set": {"meta.undo": True}} - ) + back.undo_new_following(MY_PERSON, obj) app.logger.info(f"recipients={recipients}") activity = ap.clean_activity(activity.to_dict()) @@ -2349,9 +2345,7 @@ def task_finish_post_to_inbox(): elif obj.has_type(ap.ActivityType.ANNOUNCE): back.inbox_undo_announce(MY_PERSON, obj) elif obj.has_type(ap.ActivityType.FOLLOW): - DB.activities.update_one( - {"remote_id": obj.id}, {"$set": {"meta.undo": True}} - ) + back.undo_new_follower(MY_PERSON, obj) try: invalidate_cache(activity) except Exception: @@ -2373,7 +2367,7 @@ def post_to_outbox(activity: ap.BaseActivity) -> str: obj_id = back.random_object_id() activity.set_id(back.activity_url(obj_id), obj_id) - save(Box.OUTBOX, activity) + back.save(Box.OUTBOX, activity) Tasks.cache_actor(activity.id) Tasks.finish_post_to_outbox(activity.id) return activity.id @@ -2382,14 +2376,7 @@ def post_to_outbox(activity: ap.BaseActivity) -> str: def post_to_inbox(activity: ap.BaseActivity) -> None: # Check for Block activity actor = activity.get_actor() - if DB.activities.find_one( - { - "box": Box.OUTBOX.value, - "type": ap.ActivityType.BLOCK.value, - "activity.object": actor.id, - "meta.undo": False, - } - ): + if back.outbox_is_blocked(MY_PERSON, actor.id): app.logger.info( f"actor {actor!r} is blocked, dropping the received activity {activity!r}" ) @@ -2399,7 +2386,7 @@ def post_to_inbox(activity: ap.BaseActivity) -> None: # The activity is already in the inbox app.logger.info(f"received duplicate activity {activity!r}, dropping it") - save(Box.INBOX, activity) + back.save(Box.INBOX, activity) Tasks.process_new_activity(activity.id) app.logger.info(f"spawning task for {activity!r}") @@ -2667,7 +2654,7 @@ def task_forward_activity(): iri = task.payload try: activity = ap.fetch_remote_activity(iri) - recipients = followers_as_recipients() + recipients = back.followers_as_recipients() app.logger.debug(f"Forwarding {activity!r} to {recipients}") activity = ap.clean_activity(activity.to_dict()) payload = json.dumps(activity) diff --git a/tasks.py b/tasks.py index 519dd48..f34d217 100644 --- a/tasks.py +++ b/tasks.py @@ -312,7 +312,7 @@ def post_to_inbox(activity: ap.BaseActivity) -> None: # The activity is already in the inbox log.info(f"received duplicate activity {activity!r}, dropping it") - activitypub.save(Box.INBOX, activity) + back.save(Box.INBOX, activity) process_new_activity.delay(activity.id) log.info(f"spawning task for {activity!r}") @@ -387,7 +387,7 @@ def post_to_outbox(activity: ap.BaseActivity) -> str: obj_id = back.random_object_id() activity.set_id(back.activity_url(obj_id), obj_id) - activitypub.save(Box.OUTBOX, activity) + back.save(Box.OUTBOX, activity) cache_actor.delay(activity.id) finish_post_to_outbox.delay(activity.id) return activity.id @@ -440,7 +440,7 @@ def finish_post_to_outbox(self, iri: str) -> None: def forward_activity(self, iri: str) -> None: try: activity = ap.fetch_remote_activity(iri) - recipients = activitypub.followers_as_recipients() + recipients = back.followers_as_recipients() log.debug(f"Forwarding {activity!r} to {recipients}") activity = ap.clean_activity(activity.to_dict()) payload = json.dumps(activity) From 686d614386874f6aa45d16e8c121da537965cbd6 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 8 Apr 2019 20:56:12 +0200 Subject: [PATCH 0375/1425] Tweak the admin --- app.py | 1 + templates/admin_tasks.html | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/app.py b/app.py index 00580bc..45872e1 100644 --- a/app.py +++ b/app.py @@ -1303,6 +1303,7 @@ def admin(): def admin_tasks(): return render_template( "admin_tasks.html", + success=p.get_success(), dead=p.get_dead(), waiting=p.get_waiting(), cron=p.get_cron(), diff --git a/templates/admin_tasks.html b/templates/admin_tasks.html index d0b300e..e336f86 100644 --- a/templates/admin_tasks.html +++ b/templates/admin_tasks.html @@ -84,6 +84,32 @@
              {{ task.task_id }}{{ task.url }} ({{ task.expected }}{{ task.url }} ({{ task.expected }}) {{ task.payload }} {{ task.next_run }} Tries #{{ task.tries }}: {{ task.last_error_body }} ({{ task.last_error_status_code }})
              {{ task.task_id }} {{ task.url }} ({{ task.expected }})
              {{ task.task_id }}{{ task.url }} ({{ task.expected }}{{ task.url }} ({{ task.expected }}) {{ task.payload }} {{ task.next_run }} Tries #{{ task.tries }}: {{ task.last_error_body }} ({{ task.last_error_status_code }})
              +

              Success

              + + + + + + + + + + + + + {% for task in success %} + + + + + + + + {% endfor %} + +
              #URLPayloadNext runResponse
              {{ task.task_id }}{{ task.url }} ({{ task.expected }}){{ task.payload }}{{ task.next_run }}Tries #{{ task.tries }}: {{ task.last_error_body }} ({{ task.last_error_status_code }})
              + +
              • From cd8662f04d2d9c0c71ddbd3550bcce3359479b92 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 9 Apr 2019 08:40:48 +0200 Subject: [PATCH 0376/1425] Tweak cleanup tasks --- app.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index 45872e1..13f5495 100644 --- a/app.py +++ b/app.py @@ -1390,7 +1390,9 @@ def admin_new(): @login_required def admin_notifications(): # Setup the cron for deleting old activities - p.push({}, "/task/cleanup", schedule="@every 12h") + + # FIXME(tsileo): put back to 12h + p.push({}, "/task/cleanup", schedule="@every 1h") # Trigger a cleanup if asked if request.args.get("cleanup"): @@ -2847,11 +2849,11 @@ def task_cleanup_part_2(): "meta.keep": False, "activity.published": {"$lt": d}, } - ): + ).limit(5000): # Delete the cached attachment/ for grid_item in MEDIA_CACHE.fs.find({"remote_id": data["remote_id"]}): MEDIA_CACHE.fs.delete(grid_item._id) - DB.activities.delete_one({"_id": data["_id"]}) + DB.activities.delete_one({"_id": data["_id"]}) return "OK" From 243fcd8ca6f758bd7b59a70f170bc4796df4e928 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 9 Apr 2019 21:32:16 +0200 Subject: [PATCH 0377/1425] Run MongoDB compaction on startup --- config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/config.py b/config.py index 846ae79..3be15f2 100644 --- a/config.py +++ b/config.py @@ -103,6 +103,7 @@ MEDIA_CACHE = MediaCache(GRIDFS, USER_AGENT) def create_indexes(): + DB.command("compact", "activities") DB.activities.create_index([("remote_id", pymongo.ASCENDING)]) DB.activities.create_index([("activity.object.id", pymongo.ASCENDING)]) DB.activities.create_index( From e01891d364956856ca3167806e0a0cdb6861bbd1 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 9 Apr 2019 21:32:40 +0200 Subject: [PATCH 0378/1425] Add read-only support for poll/Question --- templates/utils.html | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/templates/utils.html b/templates/utils.html index 3f54f3d..bd5ca6e 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -56,6 +56,13 @@ {% if obj | has_type('Article') %} {{ obj.name }} {{ obj | url_or_id | get_url }} + {% elif obj | has_type('Question') %} + {{ obj.content | clean | safe }} +
                + {% for oneOf in obj.oneOf %} +
              • {{ oneOf.name }} ({{ oneOf.replies.totalItems }})
                • + {% endfor %} +
              {% else %} {{ obj.content | clean | safe }} {% endif %} From c100796c8617659e33af3a369b71226c1d776122 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 10 Apr 2019 22:50:36 +0200 Subject: [PATCH 0379/1425] Emojis \o/ and fix threads --- activitypub.py | 1 + app.py | 28 ++++++++++++++-------------- config.py | 7 +++++++ templates/layout.html | 16 +++++++++++++++- templates/new.html | 36 ++++++++++++++++++++++++++++++++++-- 5 files changed, 71 insertions(+), 17 deletions(-) diff --git a/activitypub.py b/activitypub.py index 9185745..4f7104a 100644 --- a/activitypub.py +++ b/activitypub.py @@ -473,6 +473,7 @@ class MicroblogPubBackend(Backend): # It means the activity is not in the inbox, and not in the outbox, we want to save it self.save(Box.REPLIES, reply) new_threads.append(reply.id) + # TODO(tsileo): parses the replies collection and import the replies? while reply is not None: in_reply_to = reply.inReplyTo diff --git a/app.py b/app.py index 13f5495..debe25c 100644 --- a/app.py +++ b/app.py @@ -72,6 +72,7 @@ from config import BASE_URL from config import DB from config import DEBUG_MODE from config import DOMAIN +from config import EMOJIS from config import HEADERS from config import ICON_URL from config import ID @@ -91,6 +92,7 @@ from poussetaches import PousseTaches phost = "http://" + os.getenv("COMPOSE_PROJECT_NAME", "") p = PousseTaches(f"{phost}_poussetaches_1:7991", f"{phost}_web_1:5005") +# p = PousseTaches("http://localhost:7991", "http://localhost:5000") back = activitypub.MicroblogPubBackend() ap.use_backend(back) @@ -831,25 +833,22 @@ def with_replies(): ) -def _build_thread(data, include_children=True): +def _build_thread(data, include_children=True): # noqa: C901 data["_requested"] = True - print(data) root_id = data["meta"].get("thread_root_parent", data["activity"]["object"]["id"]) query = { - "$or": [ - {"meta.thread_root_parent": root_id, "type": "Create"}, - {"activity.object.id": root_id}, - ] + "meta.thread_root_parent": root_id, + "meta.deleted": False, } - if data["activity"]["object"].get("inReplyTo"): - query["$or"].append( - {"activity.object.id": data["activity"]["object"]["inReplyTo"]} - ) - - # Fetch the root replies, and the children - replies = [data] + list(DB.activities.find(query)) + replies = [data] + for dat in DB.activities.find(query): + if dat["type"][0] != ActivityType.CREATE.value: + # Make a Note/Question/... looks like a Create + dat = {"activity": {"object": dat["activity"]}, "meta": dat["meta"], "_id": dat["_id"]} + replies.append(dat) replies = sorted(replies, key=lambda d: d["activity"]["object"]["published"]) + # Index all the IDs in order to build a tree idx = {} replies2 = [] @@ -1326,6 +1325,7 @@ def admin_lookup(): ) print(data) + app.logger.debug(data.to_dict()) return render_template( "lookup.html", data=data, meta=meta, url=request.form.get("url") ) @@ -1383,7 +1383,7 @@ def admin_new(): content = f"@{actor.preferredUsername}@{domain} " thread = _build_thread(data) - return render_template("new.html", reply=reply_id, content=content, thread=thread) + return render_template("new.html", reply=reply_id, content=content, thread=thread, emojis=EMOJIS.split(" ")) @app.route("/admin/notifications") diff --git a/config.py b/config.py index 3be15f2..074c098 100644 --- a/config.py +++ b/config.py @@ -106,6 +106,10 @@ def create_indexes(): DB.command("compact", "activities") DB.activities.create_index([("remote_id", pymongo.ASCENDING)]) DB.activities.create_index([("activity.object.id", pymongo.ASCENDING)]) + DB.activities.create_index([("meta.thread_root_parent", pymongo.ASCENDING)]) + DB.activities.create_index( + [("meta.thread_root_parent", pymongo.ASCENDING), ("meta.deleted", pymongo.ASCENDING)] + ) DB.activities.create_index( [("activity.object.id", pymongo.ASCENDING), ("meta.deleted", pymongo.ASCENDING)] ) @@ -195,3 +199,6 @@ ME = { }, "publicKey": KEY.to_dict(), } + +# TODO(tsileo): read the config from the YAML if set +EMOJIS = "😺 😸 😹 😻 😼 😽 🙀 😿 😾 😺 😸 😹 😻 😼 😽 🙀 😿 😾" diff --git a/templates/layout.html b/templates/layout.html index fb7d6d2..eab7301 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -11,7 +11,11 @@ {% if not request.args.get("older_than") and not request.args.get("previous_than") %}{% endif %} {% block links %}{% endblock %} {% if config.THEME_COLOR %}{% endif %} - + {% block headers %}{% endblock %} @@ -39,5 +43,15 @@ Powered by microblog.pub {{ microblogpub_version }} (source code) and the ActivityPub protocol
          + + diff --git a/templates/new.html b/templates/new.html index a14bec7..04e34f6 100644 --- a/templates/new.html +++ b/templates/new.html @@ -1,6 +1,9 @@ {% extends "layout.html" %} {% import 'utils.html' as utils %} {% block title %}New - {{ config.NAME }}{% endblock %} +{% block headers %} + +{% endblock %} {% block content %}
          {% include "header.html" %} @@ -15,7 +18,14 @@ {% if reply %}{% endif %} - + +

          +{% for emoji in emojis %} +{{ emoji }} +{% endfor %} +

          + +
          @@ -24,4 +34,26 @@
          -{% endblock %} +{% endblock %} From 4122912c19cda96618d66e72515a4b437bb49bb1 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 10 Apr 2019 22:57:20 +0200 Subject: [PATCH 0380/1425] Tweak the emoji picker --- templates/new.html | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/templates/new.html b/templates/new.html index 04e34f6..490a6f5 100644 --- a/templates/new.html +++ b/templates/new.html @@ -35,23 +35,34 @@
          - From 8fafad4e37f9e5ef1c25bd4b1e4a12269f69f24b Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 11 May 2019 21:20:49 +0200 Subject: [PATCH 0461/1425] Tweak the README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c10c6ce..3f5adad 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ Getting closer to a stable release, it should be the "last" migration. - Allows you to attach files to your notes - Privacy-aware image upload endpoint that strip EXIF meta data before storing the file - No JavaScript, **that's it**. Even the admin UI is pure HTML/CSS + - (well except for the Emoji picker within the admin, but it's only few line of hand-written JavaScript) - Easy to customize (the theme is written Sass) - mobile-friendly theme - with dark and light version From ffd06f946b2c6175ea15ba034a140f6c2b341937 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 11 May 2019 22:52:51 +0200 Subject: [PATCH 0462/1425] Fix formatting --- app.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/app.py b/app.py index 9f5b112..3a01e3a 100644 --- a/app.py +++ b/app.py @@ -243,8 +243,7 @@ def _get_file_url(url, size, kind): @app.template_filter() def emojify(text): return emoji_unicode.replace( - text, - lambda e: EMOJI.format(filename=e.code_points, raw=e.unicode), + text, lambda e: EMOJI.format(filename=e.code_points, raw=e.unicode) ) @@ -315,8 +314,7 @@ def is_from_outbox(t): def clean(html): out = clean_html(html) return emoji_unicode.replace( - out, - lambda e: EMOJI.format(filename=e.code_points, raw=e.unicode), + out, lambda e: EMOJI.format(filename=e.code_points, raw=e.unicode) ) From 160136acdfe724f1448e82a0ea19ed8039f1fe48 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 12 May 2019 10:02:28 +0200 Subject: [PATCH 0463/1425] Add the special trash collection --- app.py | 75 ++++++++++++++++++++++++++++++++++--------------------- config.py | 3 +++ 2 files changed, 50 insertions(+), 28 deletions(-) diff --git a/app.py b/app.py index 3a01e3a..ee9507b 100644 --- a/app.py +++ b/app.py @@ -522,6 +522,23 @@ def jsonify(**data): ) +def _get_ip(): + """Guess the IP address from the request. Only used for security purpose (failed logins or bad payload). + + Geoip will be returned if the "broxy" headers are set (it does Geoip + using an offline database and append these special headers). + """ + ip = request.headers.get("X-Forwarded-For", request.remote_addr) + geoip = None + if request.headers.get("Broxy-Geoip-Country"): + geoip = ( + request.headers.get("Broxy-Geoip-Country") + + "/" + + request.headers.get("Broxy-Geoip-Region") + ) + return ip, geoip + + def is_api_request(): h = request.headers.get("Accept") if h is None: @@ -1733,7 +1750,19 @@ def inbox(): ) ) - data = request.get_json(force=True) + try: + data = request.get_json(force=True) + except Exception: + return Response( + status=422, + headers={"Content-Type": "application/json"}, + response=json.dumps( + { + "error": "failed to decode request as JSON" + } + ), + ) + print(f"req_headers={request.headers}") print(f"raw_data={data}") logger.debug(f"req_headers={request.headers}") @@ -1755,24 +1784,26 @@ def inbox(): if data["type"] == ActivityType.DELETE.value and data["id"].startswith( data["object"] ): - logger.info(f"received a Delete for an actor {data!r}") - if get_backend().inbox_check_duplicate(MY_PERSON, data["id"]): - # The activity is already in the inbox - logger.info(f"received duplicate activity {data!r}, dropping it") + # If we're here, this means the key is not saved, so we cannot verify the object + logger.info(f"received a Delete for an unknown actor {data!r}, drop it") - DB.activities.insert_one( - { - "box": Box.INBOX.value, - "activity": data, - "type": _to_list(data["type"]), - "remote_id": data["id"], - "meta": {"undo": False, "deleted": False}, - } - ) - # TODO(tsileo): write the callback the the delete external actor event return Response(status=201) except Exception: - logger.exception(f'failed to fetch remote id at {data["id"]}') + logger.exception(f'failed to fetch remote for payload {data!r}') + + # Track/store the payload for analysis + ip, geoip = _get_ip() + + DB.trash.insert({ + "activity": data, + "meta": { + "ts": datetime.now().timestamp(), + "ip_address": ip, + "geoip": geoip, + "tb": traceback.format_exc(), + }, + }) + return Response( status=422, headers={"Content-Type": "application/json"}, @@ -2196,18 +2227,6 @@ def indieauth_flow(): return redirect(red) -def _get_ip(): - ip = request.headers.get("X-Forwarded-For", request.remote_addr) - geoip = None - if request.headers.get("Broxy-Geoip-Country"): - geoip = ( - request.headers.get("Broxy-Geoip-Country") - + "/" - + request.headers.get("Broxy-Geoip-Region") - ) - return ip, geoip - - @app.route("/indieauth", methods=["GET", "POST"]) def indieauth_endpoint(): if request.method == "GET": diff --git a/config.py b/config.py index 3f35d2f..817bacf 100644 --- a/config.py +++ b/config.py @@ -103,6 +103,9 @@ MEDIA_CACHE = MediaCache(GRIDFS, USER_AGENT) def create_indexes(): + if "trash" not in DB.collection_names(): + DB.create_collection("trash", capped=True, size=50 << 20) # 50 MB + DB.command("compact", "activities") DB.activities.create_index([("remote_id", pymongo.ASCENDING)]) DB.activities.create_index([("activity.object.id", pymongo.ASCENDING)]) From 31d356ea49e1973452a01bd23a5fb2bad5a08e01 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 12 May 2019 10:06:26 +0200 Subject: [PATCH 0464/1425] Tweak the trash --- app.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/app.py b/app.py index ee9507b..e7c98e6 100644 --- a/app.py +++ b/app.py @@ -1756,11 +1756,7 @@ def inbox(): return Response( status=422, headers={"Content-Type": "application/json"}, - response=json.dumps( - { - "error": "failed to decode request as JSON" - } - ), + response=json.dumps({"error": "failed to decode request as JSON"}), ) print(f"req_headers={request.headers}") @@ -1789,20 +1785,23 @@ def inbox(): return Response(status=201) except Exception: - logger.exception(f'failed to fetch remote for payload {data!r}') + logger.exception(f"failed to fetch remote for payload {data!r}") # Track/store the payload for analysis ip, geoip = _get_ip() - DB.trash.insert({ - "activity": data, - "meta": { - "ts": datetime.now().timestamp(), - "ip_address": ip, - "geoip": geoip, - "tb": traceback.format_exc(), - }, - }) + DB.trash.insert( + { + "activity": data, + "meta": { + "ts": datetime.now().timestamp(), + "ip_address": ip, + "geoip": geoip, + "tb": traceback.format_exc(), + "headers": dict(request.headers), + }, + } + ) return Response( status=422, From 0f6fa36fbdc014bcaa6be0c4404acaa43b6701a9 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 12 May 2019 13:05:27 +0200 Subject: [PATCH 0465/1425] Make the emoji template configurable --- app.py | 7 +++---- config.py | 8 +++++++- templates/new.html | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/app.py b/app.py index e7c98e6..a899489 100644 --- a/app.py +++ b/app.py @@ -69,6 +69,7 @@ from config import BASE_URL from config import DB from config import DEBUG_MODE from config import DOMAIN +from config import EMOJI_TPL from config import EMOJIS from config import HEADERS from config import ICON_URL @@ -88,8 +89,6 @@ from utils.key import get_secret_key from utils.lookup import lookup from utils.media import Kind -EMOJI = '{raw}' - p = PousseTaches( os.getenv("MICROBLOGPUB_POUSSETACHES_HOST", "http://localhost:7991"), os.getenv("MICROBLOGPUB_INTERNAL_HOST", "http://localhost:5000"), @@ -243,7 +242,7 @@ def _get_file_url(url, size, kind): @app.template_filter() def emojify(text): return emoji_unicode.replace( - text, lambda e: EMOJI.format(filename=e.code_points, raw=e.unicode) + text, lambda e: EMOJI_TPL.format(filename=e.code_points, raw=e.unicode) ) @@ -314,7 +313,7 @@ def is_from_outbox(t): def clean(html): out = clean_html(html) return emoji_unicode.replace( - out, lambda e: EMOJI.format(filename=e.code_points, raw=e.unicode) + out, lambda e: EMOJI_TPL.format(filename=e.code_points, raw=e.unicode) ) diff --git a/config.py b/config.py index 817bacf..bec0202 100644 --- a/config.py +++ b/config.py @@ -219,5 +219,11 @@ ME = { "publicKey": KEY.to_dict(), } -# TODO(tsileo): read the config from the YAML if set EMOJIS = "😺 😸 😹 😻 😼 😽 🙀 😿 😾" +if conf.get("emojis"): + EMOJIS = conf["emojis"] + +# Emoji template for the FE +EMOJI_TPL = '{raw}' +if conf.get("emoji_tpl"): + EMOJI_TPL = conf["emoji_tpl"] diff --git a/templates/new.html b/templates/new.html index ddd16c5..a03cc62 100644 --- a/templates/new.html +++ b/templates/new.html @@ -25,7 +25,7 @@

          {% for emoji in emojis %} -{{ emoji }} +{{ emoji | emojify | safe }} {% endfor %}

          From 766678fb8f416f54e8b88dd25693d9cc7c38b5e5 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 12 May 2019 18:59:24 +0200 Subject: [PATCH 0466/1425] Fix the IndieAuth template log --- templates/admin_indieauth.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/admin_indieauth.html b/templates/admin_indieauth.html index 85000dd..510ec9c 100644 --- a/templates/admin_indieauth.html +++ b/templates/admin_indieauth.html @@ -9,7 +9,7 @@
            {% for action in indieauth_actions %}
          • {{action.ts|format_ts}} -{% if action.verified_by == "login" %}Authentication {% else %}Authorization {% endif %} +{% if action.verified_by == "id" %}Authentication {% else %}Authorization {% endif %} request by {{ action.client_id }} / {{action.ip_address}} {% if action.geoip %}({{action.geoip}}){% endif %} as {{action.me}} From 42bf96e44afa0694e5f208c1ad8e239d8db8bb61 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 17 May 2019 18:54:03 +0200 Subject: [PATCH 0467/1425] Fix the setup wizard (fixes #51) --- setup_wizard/wizard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup_wizard/wizard.py b/setup_wizard/wizard.py index e38d186..dfc7199 100644 --- a/setup_wizard/wizard.py +++ b/setup_wizard/wizard.py @@ -28,7 +28,7 @@ def main() -> None: print("Your identity will be @{username}@{domain}") dat["domain"] = prompt("domain: ") dat["username"] = prompt("username: ") - dat["password"] = bcrypt.hashpw( + dat["pass"] = bcrypt.hashpw( prompt("password: ", is_password=True).encode(), bcrypt.gensalt() ).decode() dat["name"] = prompt("name (e.g. John Doe): ") From f8d341f94a05b877124422a7c2fd6d24a790faca Mon Sep 17 00:00:00 2001 From: Jack Laxson Date: Mon, 17 Jun 2019 16:55:39 -0400 Subject: [PATCH 0468/1425] wizard fixup (#55) * Retool dockerfile to stick to 3.7 & add maintainer, change wizard to not default always to "app", run when .env exists (like it does in the git tree) * don't mix up microblog.pub v. micropub --- Makefile | 4 ++-- setup_wizard/Dockerfile | 4 +++- setup_wizard/wizard.py | 9 +++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 656a651..0f56b7a 100644 --- a/Makefile +++ b/Makefile @@ -2,13 +2,13 @@ PYTHON=python SETUP_WIZARD_IMAGE=microblogpub-setup-wizard:latest PWD=$(shell pwd) -# Build the config (will error if an existing config is found) via a Docker container +# Build the config (will error if an existing config/me.yml is found) via a Docker container .PHONY: config config: # Build the container for the setup wizard on-the-fly cd setup_wizard && docker build . -t $(SETUP_WIZARD_IMAGE) # Run and remove instantly - -docker run --rm -it --volume $(PWD):/app/out $(SETUP_WIZARD_IMAGE) + -docker run -e MICROBLOGPUB_WIZARD_PROJECT_NAME --rm -it --volume $(PWD):/app/out $(SETUP_WIZARD_IMAGE) # Finally, remove the tagged image docker rmi $(SETUP_WIZARD_IMAGE) diff --git a/setup_wizard/Dockerfile b/setup_wizard/Dockerfile index 4890f07..83b54b3 100644 --- a/setup_wizard/Dockerfile +++ b/setup_wizard/Dockerfile @@ -1,5 +1,7 @@ -FROM python:3 +FROM python:3.7 WORKDIR /app ADD . /app RUN pip install -r requirements.txt +LABEL maintainer="t@a4.io" +LABEL pub.microblog.oneshot=true CMD ["python", "wizard.py"] diff --git a/setup_wizard/wizard.py b/setup_wizard/wizard.py index dfc7199..98b9676 100644 --- a/setup_wizard/wizard.py +++ b/setup_wizard/wizard.py @@ -15,12 +15,11 @@ def main() -> None: config_file = Path("/app/out/config/me.yml") env_file = Path("/app/out/.env") - if config_file.exists() or env_file.exists(): + if config_file.exists(): # Spit out the relative path for the "config artifacts" rconfig_file = "config/me.yml" - renv_file = ".env" print( - f"Existing setup detected, please delete {rconfig_file} and/or {renv_file} before restarting the wizard" + f"Existing setup detected, please delete {rconfig_file} before restarting the wizard" ) sys.exit(2) @@ -58,12 +57,14 @@ def main() -> None: with config_file.open("w") as f: f.write(out) + proj_name = os.getenv("MICROBLOGPUB_WIZARD_PROJECT_NAME", "microblogpub") + env = { "WEB_PORT": 5005, "CONFIG_DIR": "./config", "DATA_DIR": "./data", "POUSSETACHES_AUTH_KEY": binascii.hexlify(os.urandom(32)).decode(), - "COMPOSE_PROJECT_NAME": Path.cwd().name.replace(".", ""), + "COMPOSE_PROJECT_NAME": proj_name, } out2 = "" From 1189910b532b9a829c05537cfcd834dc56b7bacb Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 29 Jun 2019 11:33:29 +0200 Subject: [PATCH 0469/1425] Fix `Question` display Pleroma does not set the `endTime` field as Mastodon does. --- app.py | 7 +++++-- templates/lookup.html | 2 +- templates/utils.html | 9 +++++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/app.py b/app.py index a899489..7489735 100644 --- a/app.py +++ b/app.py @@ -391,7 +391,7 @@ def gt_ts(val): def format_timeago(val): if val: dt = parser.parse(val) - return timeago.format(dt, datetime.now(timezone.utc)) + return timeago.format(dt.astimezone(timezone.utc), datetime.now(timezone.utc)) return val @@ -405,7 +405,9 @@ def has_type(doc, _types): @app.template_filter() def has_actor_type(doc): - for t in ap.ACTOR_TYPES: + # FIXME(tsileo): skipping the last one "Question", cause Mastodon sends question restuls as an update coming from + # the question... Does Pleroma do that too? + for t in ap.ACTOR_TYPES[:-1]: if has_type(doc, t.value): return True return False @@ -2184,6 +2186,7 @@ def _get_prop(props, name, default=None): def get_client_id_data(url): + # FIXME(tsileo): ensure not localhost via `little_boxes.urlutils.is_url_valid` data = mf2py.parse(url=url) for item in data["items"]: if "h-x-app" in item["type"] or "h-app" in item["type"]: diff --git a/templates/lookup.html b/templates/lookup.html index ebf7213..13675bf 100644 --- a/templates/lookup.html +++ b/templates/lookup.html @@ -29,7 +29,7 @@ {{ utils.display_actor_inline(data, size=80) }} {% elif data | has_type('Create') %} {{ utils.display_note(data.object, ui=True) }} - {% elif data | has_type(['Note', 'Article', 'Video', 'Audio', 'Page']) %} + {% elif data | has_type(['Note', 'Article', 'Video', 'Audio', 'Page', 'Question']) %} {{ utils.display_note(data, ui=True) }} {% elif data | has_type('Announce') %} {% set boost_actor = meta.actor %} diff --git a/templates/utils.html b/templates/utils.html index a5b713c..ed0933e 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -38,6 +38,7 @@ {% set aid = obj.id | quote_plus %} {% endif %} +{% set real_end_time = obj.closed or obj.endTime %}
            @@ -95,7 +96,7 @@ {% endfor %}

          - {% if obj.closed or obj.endTime | gtnow %} + {% if real_end_time | gtnow %} Ended {{ obj.endTime | format_timeago }} with {{ total_votes }} vote{% if total_votes | gtone %}s{% endif %}. {% else %} Ends {{ obj.endTime | format_timeago }} ({{ total_votes }} vote{% if total_votes | gtone %}s{% endif %} as of now). @@ -108,7 +109,7 @@

        • -{% if not meta.voted_for and not obj.endTime | gtnow %} +{% if not meta.voted_for and not real_end_time | gtnow %}
          @@ -123,8 +124,8 @@
          • {% endfor %} -

          {% if obj.endTime | gtnow %}This question ended {{ obj.endTime | format_timeago }}.

          - {% else %}This question ends {{ obj.endTime | format_timeago }}{% endif %} +

          {% if real_end_time | gtnow %}This question ended {{ real_end_time | format_timeago }}.

          + {% else %}This question ends {{ real_end_time | format_timeago }}{% endif %}

        From 8280104866046d96d162d5941f372bee2b739f47 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 29 Jun 2019 11:35:37 +0200 Subject: [PATCH 0470/1425] Clean the README --- README.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/README.md b/README.md index 3f5adad..621fcee 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,7 @@

        A self-hosted, single-user, ActivityPub powered microblog.

        -**Still in early development.** - -## /!\ Note to adventurer - -If you are running an instance with Celery/RabbitMQ, you will need to [perform a migration](https://github.com/tsileo/microblog.pub/tree/drop-celery#perform-the-drop-celery-migration). - -Getting closer to a stable release, it should be the "last" migration. +**Still in early development/I do not recommend to run an instance yet.** ## Features From 63074778711a75382bff0a52604808fa113bfea8 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 29 Jun 2019 11:36:21 +0200 Subject: [PATCH 0471/1425] Tweak the README --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 621fcee..ed7afb4 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ src="https://sos-ch-dk-2.exo.io/microblogpub/microblobpub.png" width="200" height="200" border="0" alt="microblog.pub">

        +

        A self-hosted, single-user, ActivityPub powered microblog.

        Build Status #microblog.pub on Matrix @@ -12,9 +13,6 @@ Code style: black

        - -

        A self-hosted, single-user, ActivityPub powered microblog.

        - **Still in early development/I do not recommend to run an instance yet.** ## Features From d7aa685e22693955671d1fd1b675b71e249e0d1c Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 2 Jul 2019 22:25:07 +0200 Subject: [PATCH 0472/1425] Tweak `Question` display --- app.py | 25 ++++++++++++++++++++++--- templates/utils.html | 11 +++++------ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/app.py b/app.py index 7489735..1dfb157 100644 --- a/app.py +++ b/app.py @@ -442,9 +442,28 @@ def is_img(filename): @app.template_filter() -def get_answer_count(choice, meta): - print(choice, meta) - return meta.get("question_answers", {}).get(_answer_key(choice), 0) +def get_answer_count(choice, obj, meta): + count_from_meta = meta.get("question_answers", {}).get(_answer_key(choice), 0) + print(count_from_meta) + print(choice, obj, meta) + if count_from_meta: + return count_from_meta + for option in obj.get("oneOf", obj.get("anyOf", [])): + if option.get("name") == choice: + return option.get("replies", {}).get("totalItems", 0) + + +@app.template_filter() +def get_total_answers_count(obj, meta): + cached = meta.get("question_replies", 0) + if cached: + return cached + cnt = 0 + print("OKI", obj) + for choice in obj.get("anyOf", obj.get("oneOf", [])): + print(choice) + cnt += choice.get("replies", {}).get("totalItems", 0) + return cnt def add_response_headers(headers={}): diff --git a/templates/utils.html b/templates/utils.html index ed0933e..56c5204 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -76,14 +76,13 @@ {% elif obj | has_type('Question') %} {{ obj.content | clean | safe }} - {% if obj.id | is_from_outbox or (meta.question_replies and (obj.closed or meta.voted_for)) %} + {% if obj.id | is_from_outbox or obj.closed or meta.voted_for %}
          - {% set total_votes = meta.question_replies %} - + {% set total_votes = obj | get_total_answers_count(meta) %} {% for oneOf in obj.oneOf %} {% set pct = 0 %} {% if total_votes > 0 %} - {% set cnt = oneOf.name | get_answer_count(meta) %} + {% set cnt = oneOf.name | get_answer_count(obj, meta) %} {% set pct = cnt * 100.0 / total_votes %} {% endif %}
        • @@ -97,9 +96,9 @@

        {% if real_end_time | gtnow %} - Ended {{ obj.endTime | format_timeago }} with {{ total_votes }} vote{% if total_votes | gtone %}s{% endif %}. + Ended {{ real_end_time | format_timeago }} with {{ total_votes }} vote{% if total_votes | gtone %}s{% endif %}. {% else %} - Ends {{ obj.endTime | format_timeago }} ({{ total_votes }} vote{% if total_votes | gtone %}s{% endif %} as of now). + Ends {{ real_end_time | format_timeago }} ({{ total_votes }} vote{% if total_votes | gtone %}s{% endif %} as of now). {% endif %}

        {% else %} From be80f0e84c311f0604a652ed830f3abdd5ba2d4a Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 4 Jul 2019 23:22:04 +0200 Subject: [PATCH 0473/1425] Fix theme CSS --- sass/base_theme.scss | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sass/base_theme.scss b/sass/base_theme.scss index 8628370..980fbd5 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -265,7 +265,8 @@ a:hover { background: $color-menu-background; padding: 5px; color: $color-light; - margin-right:5px; + margin-right: 10px; + float: left; border-radius:2px; } .bar-item:hover { From 6af6215082e85d6953f9a2465a801458deea643c Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 4 Jul 2019 23:22:20 +0200 Subject: [PATCH 0474/1425] Add html5lib for better parsing --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 51a17b5..2d749d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,3 +22,4 @@ pyyaml pillow cachetools emoji-unicode +html5lib From 0b3d4251de8cb5a843d9d155b88d7ad3783cb715 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 4 Jul 2019 23:22:38 +0200 Subject: [PATCH 0475/1425] Improve poll/question support --- activitypub.py | 19 +++++- app.py | 132 +++++++++++++++++++++++++----------------- templates/new.html | 1 + templates/stream.html | 8 ++- templates/utils.html | 41 ++++--------- 5 files changed, 115 insertions(+), 86 deletions(-) diff --git a/activitypub.py b/activitypub.py index 42478d6..0d4a5c2 100644 --- a/activitypub.py +++ b/activitypub.py @@ -3,12 +3,14 @@ import json import logging import os from datetime import datetime +from datetime import timezone from enum import Enum from typing import Any from typing import Dict from typing import List from typing import Optional +from dateutil import parser from bson.objectid import ObjectId from cachetools import LRUCache from feedgen.feed import FeedGenerator @@ -28,6 +30,7 @@ from config import ID from config import ME from config import USER_AGENT from config import USERNAME +from tasks import Tasks logger = logging.getLogger(__name__) @@ -211,7 +214,7 @@ class MicroblogPubBackend(Backend): logger.info(f"dereference {iri} via HTTP") return super().fetch_iri(iri) - def fetch_iri(self, iri: str) -> ap.ObjectType: + def fetch_iri(self, iri: str, no_cache=False) -> ap.ObjectType: if iri == ME["id"]: return ME @@ -225,8 +228,11 @@ class MicroblogPubBackend(Backend): # logger.info(f"{iri} found in DB cache") # ACTORS_CACHE[iri] = data["data"] # return data["data"] + if not no_cache: + data = self._fetch_iri(iri) + else: + return super().fetch_iri(iri) - data = self._fetch_iri(iri) logger.debug(f"_fetch_iri({iri!r}) == {data!r}") if ap._has_type(data["type"], ap.ACTOR_TYPES): logger.debug(f"caching actor {iri}") @@ -468,6 +474,15 @@ class MicroblogPubBackend(Backend): @ensure_it_is_me def inbox_create(self, as_actor: ap.Person, create: ap.Create) -> None: + # If it's a `Quesiion`, trigger an async task for updating it later (by fetching the remote and updating the + # local copy) + question = create.get_object() + if question.has_type(ap.ActivityType.QUESTION): + now = datetime.now(timezone.utc) + dt = parser.parse(question.closed or question.endTime) + minutes = int((dt - now).total_seconds() / 60) + Tasks.fetch_remote_question(create.id, minutes) + self._handle_replies(as_actor, create) @ensure_it_is_me diff --git a/app.py b/app.py index 1dfb157..e27e6bd 100644 --- a/app.py +++ b/app.py @@ -88,6 +88,7 @@ from utils import opengraph from utils.key import get_secret_key from utils.lookup import lookup from utils.media import Kind +from tasks import Tasks p = PousseTaches( os.getenv("MICROBLOGPUB_POUSSETACHES_HOST", "http://localhost:7991"), @@ -1167,7 +1168,7 @@ def remove_context(activity: Dict[str, Any]) -> Dict[str, Any]: return activity -def _add_answers_to_questions(raw_doc: Dict[str, Any]) -> None: +def _add_answers_to_question(raw_doc: Dict[str, Any]) -> None: activity = raw_doc["activity"] if ( ap._has_type(activity["type"], ActivityType.CREATE) @@ -1182,7 +1183,7 @@ def _add_answers_to_questions(raw_doc: Dict[str, Any]) -> None: .get(_answer_key(choice["name"]), 0), } now = datetime.now().astimezone() - if format_datetime(now) > activity["object"]["endTime"]: + if format_datetime(now) >= activity["object"]["endTime"]: activity["object"]["closed"] = activity["object"]["endTime"] @@ -1192,7 +1193,7 @@ def activity_from_doc(raw_doc: Dict[str, Any], embed: bool = False) -> Dict[str, # Handle Questions # TODO(tsileo): what about object embedded by ID/URL? - _add_answers_to_questions(raw_doc) + _add_answers_to_question(raw_doc) if embed: return remove_context(activity) return activity @@ -1569,6 +1570,19 @@ def admin_notifications(): ], } inbox_data, older_than, newer_than = paginated_query(DB.activities, q) + if not newer_than: + nstart = datetime.now(timezone.utc).isoformat() + else: + nstart = inbox_data[0]["_id"].generation_time.isoformat() + if not older_than: + nend = (datetime.now(timezone.utc) - timedelta(days=15)).isoformat() + else: + nend = inbox_data[-1]["_id"].generation_time.isoformat() + print(nstart, nend) + notifs = list(DB.notifications.find({"datetime": {"$lte": nstart, "$gt": nend}}).sort("_id", -1).limit(50)) + inbox_data.extend(notifs) + inbox_data = sorted(inbox_data, reverse=True, key=lambda doc: doc["_id"].generation_time) + print(inbox_data) return render_template( "stream.html", @@ -1664,6 +1678,7 @@ def api_vote(): tag=[], inReplyTo=note.id, ) + raw_note["@context"] = config.DEFAULT_CTX note = ap.Note(**raw_note) create = note.build_create() @@ -1947,10 +1962,11 @@ def api_new_question(): break answers.append({"type": ActivityType.NOTE.value, "name": a}) + open_for = int(_user_api_arg("open_for")) choices = { "endTime": ap.format_datetime( datetime.now().astimezone() - + timedelta(minutes=int(_user_api_arg("open_for"))) + + timedelta(minutes=open_for) ) } of = _user_api_arg("of") @@ -1974,6 +1990,8 @@ def api_new_question(): create = question.build_create() create_id = post_to_outbox(create) + Tasks.update_question_outbox(create_id, open_for) + return _user_api_response(activity=create_id) @@ -2427,51 +2445,6 @@ def rss_feed(): ) -########### -# Tasks - - -class Tasks: - @staticmethod - def cache_object(iri: str) -> None: - p.push(iri, "/task/cache_object") - - @staticmethod - def cache_actor(iri: str, also_cache_attachments: bool = True) -> None: - p.push( - {"iri": iri, "also_cache_attachments": also_cache_attachments}, - "/task/cache_actor", - ) - - @staticmethod - def post_to_remote_inbox(payload: str, recp: str) -> None: - p.push({"payload": payload, "to": recp}, "/task/post_to_remote_inbox") - - @staticmethod - def forward_activity(iri: str) -> None: - p.push(iri, "/task/forward_activity") - - @staticmethod - def fetch_og_meta(iri: str) -> None: - p.push(iri, "/task/fetch_og_meta") - - @staticmethod - def process_new_activity(iri: str) -> None: - p.push(iri, "/task/process_new_activity") - - @staticmethod - def cache_attachments(iri: str) -> None: - p.push(iri, "/task/cache_attachments") - - @staticmethod - def finish_post_to_inbox(iri: str) -> None: - p.push(iri, "/task/finish_post_to_inbox") - - @staticmethod - def finish_post_to_outbox(iri: str) -> None: - p.push(iri, "/task/finish_post_to_outbox") - - @app.route("/task/fetch_og_meta", methods=["POST"]) def task_fetch_og_meta(): task = p.parse(request) @@ -2986,17 +2959,70 @@ def task_post_to_remote_inbox(): return "" +@app.route("/task/fetch_remote_question", methods=["POST"]) +def task_fetch_remote_question(): + """Fetch a remote question for implementation that does not send Update.""" + task = p.parse(request) + app.logger.info(f"task={task!r}") + iri = task.payload + try: + app.logger.info(f"Fetching remote question {iri}") + local_question = DB.activities.find_one( + {"box": Box.INBOX.value, "remote_id": iri} + ) + remote_question = get_backend().fetch_iri(iri, no_cache=True) + if local_question["meta"].get("voted_for") or local_question["meta"]["subscribed"]: + DB.notifications.insert_one({"type": "question_ended", "datetime": datetime.now(timezone.utc).isoformat(), + "activity": remote_question}) + + DB.activities.update_one( + {"remote_id": iri, "box": Box.INBOX.value}, + {"$set": {"activity": remote_question}}, + ) + + except HTTPError as err: + app.logger.exception("request failed") + if 400 >= err.response.status_code >= 499: + app.logger.info("client error, no retry") + return "" + + raise TaskError() from err + except Exception as err: + app.logger.exception("task failed") + raise TaskError() from err + + return "" + + @app.route("/task/update_question", methods=["POST"]) def task_update_question(): - """Post an activity to a remote inbox.""" + """Sends an Update.""" task = p.parse(request) app.logger.info(f"task={task!r}") iri = task.payload try: app.logger.info(f"Updating question {iri}") - # TODO(tsileo): sends an Update with the question/iri as an actor, with the updated stats (LD sig will fail?) - # but to who? followers and people who voted? but this must not be visible right? - # also sends/trigger a notification when a poll I voted for ends like Mastodon? + cc = [ID + "/followers"] + doc = DB.activities.find_one( + {"box": Box.OUTBOX.value, "remote_id": iri} + ) + _add_answers_to_question(doc) + question = ap.Question(**doc["activity"]["object"]) + + raw_update = dict( + actor=question.id, + object=question.to_dict(embed=True), + attributedTo=MY_PERSON.id, + cc=list(set(cc)), + to=[ap.AS_PUBLIC], + ) + raw_update["@context"] = config.DEFAULT_CTX + + update = ap.Update(**raw_update) + print(update) + print(update.to_dict()) + post_to_outbox(update) + except HTTPError as err: app.logger.exception("request failed") if 400 >= err.response.status_code >= 499: diff --git a/templates/new.html b/templates/new.html index a03cc62..9b27df6 100644 --- a/templates/new.html +++ b/templates/new.html @@ -35,6 +35,7 @@ {% if request.args.get("question") == "1" %}

        Open for: + + + + + {% endif %} + {{ '%0.0f'| format(pct) }}% - {{ oneOf.name }} + {{ oneOf.name }} {% if oneOf.name == meta.voted_for %}(your vote){% endif %} {% endfor %} @@ -101,35 +110,7 @@ Ends {{ real_end_time | format_timeago }} ({{ total_votes }} vote{% if total_votes | gtone %}s{% endif %} as of now). {% endif %}

        - {% else %} -
          - {% for oneOf in obj.oneOf %} -
        • - - -{% if not meta.voted_for and not real_end_time | gtnow %} -
          - - - - - -
           -{% else %} - ??? -{% endif %} -{{ oneOf.name }} {% if oneOf.name == meta.voted_for %}(your vote){% endif %} -           -
          • - {% endfor %} -

          {% if real_end_time | gtnow %}This question ended {{ real_end_time | format_timeago }}.

          - {% else %}This question ends {{ real_end_time | format_timeago }}{% endif %} -

          -
        - - - {% endif %} {% else %} {{ obj.content | clean | safe }} From 9e25cad86c2e0a161f72b9f17323e3aacb9fe448 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 4 Jul 2019 23:23:40 +0200 Subject: [PATCH 0476/1425] Oops add missing file and reformat --- tasks.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 tasks.py diff --git a/tasks.py b/tasks.py new file mode 100644 index 0000000..e3169f1 --- /dev/null +++ b/tasks.py @@ -0,0 +1,61 @@ +import os + +from poussetaches import PousseTaches + +p = PousseTaches( + os.getenv("MICROBLOGPUB_POUSSETACHES_HOST", "http://localhost:7991"), + os.getenv("MICROBLOGPUB_INTERNAL_HOST", "http://localhost:5000"), +) + + +class Tasks: + @staticmethod + def cache_object(iri: str) -> None: + p.push(iri, "/task/cache_object") + + @staticmethod + def cache_actor(iri: str, also_cache_attachments: bool = True) -> None: + p.push( + {"iri": iri, "also_cache_attachments": also_cache_attachments}, + "/task/cache_actor", + ) + + @staticmethod + def post_to_remote_inbox(payload: str, recp: str) -> None: + p.push({"payload": payload, "to": recp}, "/task/post_to_remote_inbox") + + @staticmethod + def forward_activity(iri: str) -> None: + p.push(iri, "/task/forward_activity") + + @staticmethod + def fetch_og_meta(iri: str) -> None: + p.push(iri, "/task/fetch_og_meta") + + @staticmethod + def process_new_activity(iri: str) -> None: + p.push(iri, "/task/process_new_activity") + + @staticmethod + def cache_attachments(iri: str) -> None: + p.push(iri, "/task/cache_attachments") + + @staticmethod + def finish_post_to_inbox(iri: str) -> None: + p.push(iri, "/task/finish_post_to_inbox") + + @staticmethod + def finish_post_to_outbox(iri: str) -> None: + p.push(iri, "/task/finish_post_to_outbox") + + @staticmethod + def update_question_outbox(iri: str, open_for: int) -> None: + p.push( + iri, "/task/update_question", delay=open_for + ) # XXX: delay expects minutes + + @staticmethod + def fetch_remote_question(iri: str, delay: int) -> None: + p.push( + iri, "/task/fetch_remote_question", delay=delay + ) # XXX: delay expects minutes From 2a0b3487968c3e5404686114526b40169448eae3 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 4 Jul 2019 23:24:25 +0200 Subject: [PATCH 0477/1425] Reformat files with black --- app.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/app.py b/app.py index e27e6bd..16312d7 100644 --- a/app.py +++ b/app.py @@ -1579,9 +1579,15 @@ def admin_notifications(): else: nend = inbox_data[-1]["_id"].generation_time.isoformat() print(nstart, nend) - notifs = list(DB.notifications.find({"datetime": {"$lte": nstart, "$gt": nend}}).sort("_id", -1).limit(50)) + notifs = list( + DB.notifications.find({"datetime": {"$lte": nstart, "$gt": nend}}) + .sort("_id", -1) + .limit(50) + ) inbox_data.extend(notifs) - inbox_data = sorted(inbox_data, reverse=True, key=lambda doc: doc["_id"].generation_time) + inbox_data = sorted( + inbox_data, reverse=True, key=lambda doc: doc["_id"].generation_time + ) print(inbox_data) return render_template( @@ -1965,8 +1971,7 @@ def api_new_question(): open_for = int(_user_api_arg("open_for")) choices = { "endTime": ap.format_datetime( - datetime.now().astimezone() - + timedelta(minutes=open_for) + datetime.now().astimezone() + timedelta(minutes=open_for) ) } of = _user_api_arg("of") @@ -2971,9 +2976,17 @@ def task_fetch_remote_question(): {"box": Box.INBOX.value, "remote_id": iri} ) remote_question = get_backend().fetch_iri(iri, no_cache=True) - if local_question["meta"].get("voted_for") or local_question["meta"]["subscribed"]: - DB.notifications.insert_one({"type": "question_ended", "datetime": datetime.now(timezone.utc).isoformat(), - "activity": remote_question}) + if ( + local_question["meta"].get("voted_for") + or local_question["meta"]["subscribed"] + ): + DB.notifications.insert_one( + { + "type": "question_ended", + "datetime": datetime.now(timezone.utc).isoformat(), + "activity": remote_question, + } + ) DB.activities.update_one( {"remote_id": iri, "box": Box.INBOX.value}, @@ -3003,9 +3016,7 @@ def task_update_question(): try: app.logger.info(f"Updating question {iri}") cc = [ID + "/followers"] - doc = DB.activities.find_one( - {"box": Box.OUTBOX.value, "remote_id": iri} - ) + doc = DB.activities.find_one({"box": Box.OUTBOX.value, "remote_id": iri}) _add_answers_to_question(doc) question = ap.Question(**doc["activity"]["object"]) From 1dd7c516ed583631d9d2e4becd842060f876658f Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Jul 2019 00:29:51 +0200 Subject: [PATCH 0478/1425] More poll/question tweaks --- app.py | 5 ++++- sass/base_theme.scss | 2 +- templates/utils.html | 5 +++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/app.py b/app.py index 16312d7..55e3de7 100644 --- a/app.py +++ b/app.py @@ -929,8 +929,11 @@ def _build_thread(data, include_children=True): # noqa: C901 } replies = [data] for dat in DB.activities.find(query): + print(dat["type"]) if dat["type"][0] == ActivityType.CREATE.value: replies.append(dat) + if dat["type"][0] == ActivityType.UPDATE.value: + continue else: # Make a Note/Question/... looks like a Create dat = { @@ -958,7 +961,7 @@ def _build_thread(data, include_children=True): # noqa: C901 rep_id = rep["activity"]["object"]["id"] if rep_id == root_id: continue - reply_of = ap._get_id(rep["activity"]["object"]["inReplyTo"]) + reply_of = ap._get_id(rep["activity"]["object"].get("inReplyTo")) try: idx[reply_of]["_nodes"].append(rep) except KeyError: diff --git a/sass/base_theme.scss b/sass/base_theme.scss index 980fbd5..ce16014 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -214,7 +214,7 @@ a:hover { overflow: hidden; } - .bottom-bar { margin-top:10px; } + .bottom-bar { margin-top:10px;display:inline-block; } .img-attachment { max-width:100%; diff --git a/templates/utils.html b/templates/utils.html index b224582..3d355e3 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -86,7 +86,7 @@ {% set pct = cnt * 100.0 / total_votes %} {% endif %}
      • - {% if not meta.voted_for and not (real_end_time | gtnow) and not (obj.id | is_from_outbox) %} + {% if session.logged_in and not meta.voted_for and not (real_end_time | gtnow) and not (obj.id | is_from_outbox) %}
        @@ -161,7 +161,8 @@
        -{% if perma %}{{ obj.published | format_time }} +{% if perma %} +{{ obj.published | format_time }} {% if not (obj.id | is_from_outbox) %} permalink {% endif %} From 9a037b132ad3907edb6de86f119b7132373fe6ed Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 5 Jul 2019 09:49:49 +0200 Subject: [PATCH 0479/1425] Support emojis CW --- templates/utils.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/utils.html b/templates/utils.html index 3d355e3..0c95326 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -62,7 +62,7 @@ {% endif %}
        - {% if obj.summary %}

        {{ obj.summary | clean }}

        {% endif %} + {% if obj.summary %}

        {{ obj.summary | clean | safe }}

        {% endif %} {% if obj | has_type('Video') %}
      diff --git a/templates/utils.html b/templates/utils.html index 0c95326..a63895e 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -212,6 +212,24 @@ {% endif %} +{% if meta.bookmarked or request.path == url_for("admin_bookmarks") %} +
      + + + + + +
      +{% else %} +
      + + + + +
      +{% endif %} + + {% endif %} {% if obj.id | is_from_outbox %} From 9e1bd068792e57e229d336e9a75e6154d4f00634 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 10 Jul 2019 23:47:42 +0200 Subject: [PATCH 0498/1425] Tweak admin menu --- templates/layout.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/layout.html b/templates/layout.html index 2e40ea6..08a7761 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -26,8 +26,8 @@
    • Public
    • New
    • Stream
      • -
    • Bookmarks
    • Notifications
      • +
    • Bookmarks
    • Lookup
    • Logout
    From 5b76fe65aa5cb4cba0355f7c83603f793b54bfc4 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 12 Jul 2019 00:16:51 +0200 Subject: [PATCH 0499/1425] Prevent two activities from being bookmarked at once This could happen when bookmarking via an Announce when the original Create is also in the DB. --- app.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/app.py b/app.py index fa18c66..886eda9 100644 --- a/app.py +++ b/app.py @@ -1727,12 +1727,14 @@ def api_bookmark(): undo = _user_api_arg("undo", default=None) == "yes" - DB.activities.update_one( - {"meta.object.id": note.id}, {"$set": {"meta.bookmarked": not undo}} - ) - DB.activities.update_one( + # Try to bookmark the `Create` first + if not DB.activities.update_one( {"activity.object.id": note.id}, {"$set": {"meta.bookmarked": not undo}} - ) + ).modified_count: + # Then look for the `Announce` + DB.activities.update_one( + {"meta.object.id": note.id}, {"$set": {"meta.bookmarked": not undo}} + ) return _user_api_response() From f5f4e7f9dff956f127866e66f2d0b1fc9f3122b1 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 12 Jul 2019 22:03:04 +0200 Subject: [PATCH 0500/1425] More poll/question bugfixes --- app.py | 19 +++++++++++-------- tasks.py | 5 ++--- utils/__init__.py | 15 +++++++++++++++ 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/app.py b/app.py index 886eda9..bd0cbce 100644 --- a/app.py +++ b/app.py @@ -23,7 +23,6 @@ import mf2py import requests import timeago from bson.objectid import ObjectId -from dateutil import parser from flask import Flask from flask import Response from flask import abort @@ -87,6 +86,7 @@ from config import VERSION_DATE from config import _drop_db from poussetaches import PousseTaches from tasks import Tasks +from utils import parse_datetime from utils import opengraph from utils.key import get_secret_key from utils.lookup import lookup @@ -262,7 +262,7 @@ def gtone(n): @app.template_filter() def gtnow(dtstr): - return format_datetime(datetime.now().astimezone()) > dtstr + return format_datetime(datetime.now(timezone.utc)) > dtstr @app.template_filter() @@ -381,7 +381,7 @@ def get_actor(url): @app.template_filter() def format_time(val): if val: - dt = parser.parse(val) + dt = parse_datetime(val) return datetime.strftime(dt, "%B %d, %Y, %H:%M %p") return val @@ -399,7 +399,7 @@ def gt_ts(val): @app.template_filter() def format_timeago(val): if val: - dt = parser.parse(val) + dt = parse_datetime(val) return timeago.format(dt.astimezone(timezone.utc), datetime.now(timezone.utc)) return val @@ -1195,7 +1195,7 @@ def _add_answers_to_question(raw_doc: Dict[str, Any]) -> None: .get("question_answers", {}) .get(_answer_key(choice["name"]), 0), } - now = datetime.now().astimezone() + now = datetime.now(timezone.utc) if format_datetime(now) >= activity["object"]["endTime"]: activity["object"]["closed"] = activity["object"]["endTime"] @@ -2065,7 +2065,7 @@ def api_new_question(): open_for = int(_user_api_arg("open_for")) choices = { "endTime": ap.format_datetime( - datetime.now().astimezone() + timedelta(minutes=open_for) + datetime.now(timezone.utc) + timedelta(minutes=open_for) ) } of = _user_api_arg("of") @@ -2923,8 +2923,11 @@ def task_process_new_activity(): in_reply_to = note.get_in_reply_to() # Make the note part of the stream if it's not a reply, or if it's a local reply **and** it's not a poll # answer - if (not in_reply_to or in_reply_to.startswith(ID)) and not note.has_type( - ap.ActivityType.QUESTION + # FIXME(tsileo): this will block "regular replies" to a Poll, maybe the adressing will help make the + # difference? + if not in_reply_to or ( + in_reply_to.startswith(ID) + and not note.has_type(ap.ActivityType.QUESTION) ): tag_stream = True diff --git a/tasks.py b/tasks.py index c2020b2..9855f64 100644 --- a/tasks.py +++ b/tasks.py @@ -2,8 +2,7 @@ import os from datetime import datetime from datetime import timezone -from dateutil import parser - +from utils import parse_datetime from poussetaches import PousseTaches p = PousseTaches( @@ -61,7 +60,7 @@ class Tasks: @staticmethod def fetch_remote_question(question) -> None: now = datetime.now(timezone.utc) - dt = parser.parse(question.closed or question.endTime).astimezone(timezone.utc) + dt = parse_datetime(question.closed or question.endTime) minutes = int((dt - now).total_seconds() / 60) if minutes > 0: diff --git a/utils/__init__.py b/utils/__init__.py index cdf368d..97d057c 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -1,4 +1,8 @@ import logging +from datetime import datetime +from datetime import timezone + +from dateutil import parser logger = logging.getLogger(__name__) @@ -10,3 +14,14 @@ def strtobool(s: str) -> bool: return False raise ValueError(f"cannot convert {s} to bool") + + +def parse_datetime(s :str) -> datetime: + # Parses the datetime with dateutil + dt = parser.parse(s) + + # If no TZ is set, assumes it's UTC + if not dt.tzinfo: + dt = dt.replace(tzinfo=timezone.utc) + + return dt From b7829ed70670d68b5f7e9ea670a7a6e9296b84f8 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 12 Jul 2019 23:52:21 +0200 Subject: [PATCH 0501/1425] Another poll bugfix --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index bd0cbce..93c54ea 100644 --- a/app.py +++ b/app.py @@ -3097,7 +3097,7 @@ def task_fetch_remote_question(): local_question and ( local_question["meta"].get("voted_for") - or local_question["meta"]["subscribed"] + or local_question["meta"].get("subscribed") ) and not DB.notifications.find_one({"activity.id": remote_question["id"]}) ): From 5effab8b90ef44e9b2660b89ab12c18893fee052 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 13 Jul 2019 00:14:29 +0200 Subject: [PATCH 0502/1425] Fix formatting --- utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/__init__.py b/utils/__init__.py index 97d057c..ccffdd8 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -16,7 +16,7 @@ def strtobool(s: str) -> bool: raise ValueError(f"cannot convert {s} to bool") -def parse_datetime(s :str) -> datetime: +def parse_datetime(s: str) -> datetime: # Parses the datetime with dateutil dt = parser.parse(s) From 6ab59d2e41de315f6376897c7dfeb2cd3f740c66 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 13 Jul 2019 00:28:14 +0200 Subject: [PATCH 0503/1425] Start working on post visibility --- app.py | 30 +++++++++++++++++++++++++----- templates/new.html | 5 +++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/app.py b/app.py index 93c54ea..840c1c0 100644 --- a/app.py +++ b/app.py @@ -1532,6 +1532,7 @@ def admin_new(): reply=reply_id, content=content, thread=thread, + visibility=ap.Visibility, emojis=EMOJIS.split(" "), ) @@ -1995,22 +1996,41 @@ def api_new_note(): except ValueError: pass + visibility = ap.Visibility[ + _user_api_arg("visibility", default=ap.Visibility.PUBLIC.name) + ] + content, tags = parse_markdown(source) - to = request.args.get("to") - cc = [ID + "/followers"] + + to, cc = [], [] + if visibility == ap.Visibility.PUBLIC: + to = [ap.AS_PUBLIC] + cc = [ID + "/followers"] + elif visibility == ap.Visibility.UNLISTED: + to = [ID + "/followers"] + cc = [ap.AS_PUBLIC] + elif visibility == ap.Visibility.FOLLOWERS_ONLY: + to = [ID + "/followers"] + cc = [] if _reply: reply = ap.fetch_remote_activity(_reply) - cc.append(reply.attributedTo) + if visibility == ap.Visibility.DIRECT: + to.append(reply.attributedTo) + else: + cc.append(reply.attributedTo) for tag in tags: if tag["type"] == "Mention": - cc.append(tag["href"]) + if visibility == ap.Visibility.DIRECT: + to.append(tag["href"]) + else: + cc.append(tag["href"]) raw_note = dict( attributedTo=MY_PERSON.id, cc=list(set(cc)), - to=[to if to else ap.AS_PUBLIC], + to=list(set(to)), content=content, tag=tags, source={"mediaType": "text/markdown", "content": source}, diff --git a/templates/new.html b/templates/new.html index 9b27df6..0fa98fc 100644 --- a/templates/new.html +++ b/templates/new.html @@ -21,6 +21,11 @@
    + {% if reply %}{% endif %}

    From 152bcf2b26d71f1cc6f7844575a60c374f538dbb Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 13 Jul 2019 00:38:51 +0200 Subject: [PATCH 0504/1425] Only show public stuff on the homepage --- app.py | 11 ++++++----- tasks.py | 2 +- templates/header.html | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/app.py b/app.py index 840c1c0..5468f59 100644 --- a/app.py +++ b/app.py @@ -86,8 +86,8 @@ from config import VERSION_DATE from config import _drop_db from poussetaches import PousseTaches from tasks import Tasks -from utils import parse_datetime from utils import opengraph +from utils import parse_datetime from utils.key import get_secret_key from utils.lookup import lookup from utils.media import Kind @@ -143,13 +143,13 @@ def inject_config(): notes_count = DB.activities.find( {"box": Box.OUTBOX.value, "$or": [q, {"type": "Announce", "meta.undo": False}]} ).count() + # FIXME(tsileo): rename to all_count, and remove poll answers from it with_replies_count = DB.activities.find( { "box": Box.OUTBOX.value, "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, "meta.undo": False, "meta.deleted": False, - "meta.public": True, } ).count() liked_count = DB.activities.count( @@ -875,6 +875,7 @@ def index(): "activity.object.inReplyTo": None, "meta.deleted": False, "meta.undo": False, + "meta.public": True, "$or": [{"meta.pinned": False}, {"meta.pinned": {"$exists": False}}], } print(list(DB.activities.find(q))) @@ -887,6 +888,7 @@ def index(): "type": ActivityType.CREATE.value, "meta.deleted": False, "meta.undo": False, + "meta.public": True, "meta.pinned": True, } pinned = list(DB.activities.find(q_pinned)) @@ -906,14 +908,13 @@ def index(): return resp -@app.route("/with_replies") +@app.route("/all") @login_required -def with_replies(): +def all(): q = { "box": Box.OUTBOX.value, "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, "meta.deleted": False, - "meta.public": True, "meta.undo": False, } outbox_data, older_than, newer_than = paginated_query(DB.activities, q) diff --git a/tasks.py b/tasks.py index 9855f64..9515fe0 100644 --- a/tasks.py +++ b/tasks.py @@ -2,8 +2,8 @@ import os from datetime import datetime from datetime import timezone -from utils import parse_datetime from poussetaches import PousseTaches +from utils import parse_datetime p = PousseTaches( os.getenv("MICROBLOGPUB_POUSSETACHES_HOST", "http://localhost:7991"), diff --git a/templates/header.html b/templates/header.html index 8fc7eb1..29c396d 100644 --- a/templates/header.html +++ b/templates/header.html @@ -18,7 +18,7 @@

    • Notes {{ notes_count }}
      • -{% if session.logged_in %}
    • With replies {{ with_replies_count }}
      • +{% if session.logged_in %}
    • All {{ with_replies_count }}
      • {% endif %}
    • Liked {{ liked_count }}
    • Followers {{ followers_count }}
      • From 243e28fa75a2f68e66a8840aab6daa878b551481 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 14 Jul 2019 23:57:23 +0200 Subject: [PATCH 0505/1425] Lot of migration work (better normalization), and more visibility work --- activitypub.py | 43 +++++++++++++++--- app.py | 35 ++++++++++----- migrations.py | 101 ++++++++++++++++++++++++++++++++++++++++++ run.sh | 1 + sass/base_theme.scss | 9 ++++ templates/layout.html | 1 + templates/utils.html | 13 ++++-- utils/migrations.py | 56 +++++++++++++++++++++++ 8 files changed, 239 insertions(+), 20 deletions(-) create mode 100644 migrations.py create mode 100644 utils/migrations.py diff --git a/activitypub.py b/activitypub.py index 68d6545..82a259d 100644 --- a/activitypub.py +++ b/activitypub.py @@ -8,6 +8,7 @@ from typing import Any from typing import Dict from typing import List from typing import Optional +from urllib.parse import urlparse from bson.objectid import ObjectId from cachetools import LRUCache @@ -114,9 +115,17 @@ class MicroblogPubBackend(Backend): def save(self, box: Box, activity: ap.BaseActivity) -> None: """Custom helper for saving an activity to the DB.""" - is_public = True - if activity.has_type(ap.ActivityType.CREATE) and not activity.is_public(): - is_public = False + visibility = ap.get_visibility(activity) + is_public = False + if visibility in [ap.Visibility.PUBLIC, ap.Visibility.UNLISTED]: + is_public = True + object_id = None + try: + object_id = activity.get_object_id() + except ValueError: + pass + + actor_id = activity.get_actor().id DB.activities.insert_one( { @@ -124,7 +133,16 @@ class MicroblogPubBackend(Backend): "activity": activity.to_dict(), "type": _to_list(activity.type), "remote_id": activity.id, - "meta": {"undo": False, "deleted": False, "public": is_public}, + "meta": { + "undo": False, + "deleted": False, + "public": is_public, + "server": urlparse(activity.id).netloc, + "visibility": visibility.name, + "actor_id": actor_id, + "object_id": object_id, + "poll_answer": False, + }, } ) @@ -481,6 +499,15 @@ class MicroblogPubBackend(Backend): @ensure_it_is_me def outbox_create(self, as_actor: ap.Person, create: ap.Create) -> None: + obj = create.get_object() + + # Flag the activity as a poll answer if needed + print(f"POLL ANSWER ChECK {obj.get_in_reply_to()} {obj.name} {obj.content}") + if obj.get_in_reply_to() and obj.name and not obj.content: + DB.activities.update_one( + {"remote_id": create.id}, {"$set": {"meta.poll_answer": True}} + ) + self._handle_replies(as_actor, create) @ensure_it_is_me @@ -540,7 +567,13 @@ class MicroblogPubBackend(Backend): DB.activities.update_one( {"remote_id": create.id}, - {"$set": {"meta.answer_to": question.id, "meta.stream": False}}, + { + "$set": { + "meta.answer_to": question.id, + "meta.stream": False, + "meta.poll_answer": True, + } + }, ) return None diff --git a/app.py b/app.py index 5468f59..60ea41a 100644 --- a/app.py +++ b/app.py @@ -144,14 +144,13 @@ def inject_config(): {"box": Box.OUTBOX.value, "$or": [q, {"type": "Announce", "meta.undo": False}]} ).count() # FIXME(tsileo): rename to all_count, and remove poll answers from it - with_replies_count = DB.activities.find( - { - "box": Box.OUTBOX.value, - "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, - "meta.undo": False, - "meta.deleted": False, - } - ).count() + all_q = { + "box": Box.OUTBOX.value, + "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, + "meta.undo": False, + "meta.deleted": False, + "meta.poll_answer": False, + } liked_count = DB.activities.count( { "box": Box.OUTBOX.value, @@ -181,7 +180,7 @@ def inject_config(): following_count=DB.activities.count(following_q) if logged_in else 0, notes_count=notes_count, liked_count=liked_count, - with_replies_count=with_replies_count if logged_in else 0, + with_replies_count=DB.activities.count(all_q) if logged_in else 0, me=ME, base_url=config.BASE_URL, ) @@ -916,6 +915,7 @@ def all(): "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, "meta.deleted": False, "meta.undo": False, + "meta.poll_answer": False, } outbox_data, older_than, newer_than = paginated_query(DB.activities, q) @@ -1218,7 +1218,7 @@ def outbox(): if request.method == "GET": if not is_api_request(): abort(404) - # TODO(tsileo): returns the whole outbox if authenticated + # TODO(tsileo): returns the whole outbox if authenticated and look at OCAP support q = { "box": Box.OUTBOX.value, "meta.deleted": False, @@ -1253,7 +1253,11 @@ def outbox(): @app.route("/outbox/") def outbox_detail(item_id): doc = DB.activities.find_one( - {"box": Box.OUTBOX.value, "remote_id": back.activity_url(item_id)} + { + "box": Box.OUTBOX.value, + "remote_id": back.activity_url(item_id), + "meta.public": True, + } ) if not doc: abort(404) @@ -1269,7 +1273,11 @@ def outbox_detail(item_id): @app.route("/outbox//activity") def outbox_activity(item_id): data = DB.activities.find_one( - {"box": Box.OUTBOX.value, "remote_id": back.activity_url(item_id)} + { + "box": Box.OUTBOX.value, + "remote_id": back.activity_url(item_id), + "meta.public": True, + } ) if not data: abort(404) @@ -1295,6 +1303,7 @@ def outbox_activity_replies(item_id): "box": Box.OUTBOX.value, "remote_id": back.activity_url(item_id), "meta.deleted": False, + "meta.public": True, } ) if not data: @@ -1305,6 +1314,7 @@ def outbox_activity_replies(item_id): q = { "meta.deleted": False, + "meta.public": True, "type": ActivityType.CREATE.value, "activity.object.inReplyTo": obj.get_object().id, } @@ -1330,6 +1340,7 @@ def outbox_activity_likes(item_id): "box": Box.OUTBOX.value, "remote_id": back.activity_url(item_id), "meta.deleted": False, + "meta.public": True, } ) if not data: diff --git a/migrations.py b/migrations.py new file mode 100644 index 0000000..1ec997f --- /dev/null +++ b/migrations.py @@ -0,0 +1,101 @@ +"""Migrations that will be run automatically at startup.""" +from typing import Any +from typing import Dict +from urllib.parse import urlparse + +from little_boxes import activitypub as ap + +from utils.migrations import DB +from utils.migrations import Migration +from utils.migrations import logger +from utils.migrations import perform # noqa: just here for export + + +class _1_MetaMigrationt(Migration): + """Add new metadata to simplify querying.""" + + def __guess_visibility(self, data: Dict[str, Any]) -> ap.Visibility: + to = data.get("to", []) + cc = data.get("cc", []) + if ap.AS_PUBLIC in to: + return ap.Visibility.PUBLIC + elif ap.AS_PUBLIC in cc: + return ap.Visibility.UNLISTED + else: + # Uses a bit of heuristic here, it's too expensive to fetch the actor, so assume the followers + # collection has "/collection" in it (which is true for most software), and at worst, we will + # classify it as "DIRECT" which behave the same as "FOLLOWERS_ONLY" (i.e. no Announce) + followers_only = False + for item in to: + if "/followers" in item: + followers_only = True + break + if not followers_only: + for item in cc: + if "/followers" in item: + followers_only = True + break + if followers_only: + return ap.Visibility.FOLLOWERS_ONLY + + return ap.Visibility.DIRECT + + def migrate(self) -> None: # noqa: C901 # too complex + for data in DB.activities.find(): + logger.info(f"before={data}") + obj = data["activity"].get("object") + set_meta: Dict[str, Any] = {} + + # Set `meta.object_id` (str) + if not data["meta"].get("object_id"): + set_meta["meta.object_id"] = None + if obj: + if isinstance(obj, str): + set_meta["meta.object_id"] = data["activity"]["object"] + elif isinstance(obj, dict): + obj_id = obj.get("id") + if obj_id: + set_meta["meta.object_id"] = obj_id + + # Set `meta.object_visibility` (str) + if not data["meta"].get("object_visibility"): + set_meta["meta.object_visibility"] = None + object_id = data["meta"].get("object_id") or set_meta.get("meta.object_id") + if object_id: + obj = data["meta"].get("object") or data["activity"].get("object") + if isinstance(obj, dict): + set_meta["meta.object_visibility"] = self.__guess_visibility(obj).name + + # Set `meta.actor_id` (str) + if not data["meta"].get("actor_id"): + set_meta["meta.actor_id"] = None + actor = data["activity"].get("actor") + if actor: + if isinstance(actor, str): + set_meta["meta.actor_id"] = data["activity"]["actor"] + elif isinstance(actor, dict): + actor_id = actor.get("id") + if actor_id: + set_meta["meta.actor_id"] = actor_id + + # Set `meta.poll_answer` (bool) + if not data["meta"].get("poll_answer"): + set_meta["meta.poll_answer"] = False + if obj: + if isinstance(obj, dict): + if ( + obj.get("name") + and not obj.get("content") + and obj.get("inReplyTo") + ): + set_meta["meta.poll_answer"] = True + + # Set `meta.visibility` (str) + if not data["meta"].get("visibility"): + set_meta["meta.visibility"] = self.__guess_visibility(data["activity"]).name + + if not data["meta"].get("server"): + set_meta["meta.server"] = urlparse(data["remote_id"]).netloc + + logger.info(f"meta={set_meta}\n") + DB.activities.update_one({"_id": data["_id"]}, {"$set": set_meta}) diff --git a/run.sh b/run.sh index 8c9465c..313d405 100755 --- a/run.sh +++ b/run.sh @@ -1,3 +1,4 @@ #!/bin/bash +python -c "import logging; logging.basicConfig(level=logging.DEBUG); import migrations; migrations.perform()" python -c "import config; config.create_indexes()" gunicorn -t 600 -w 5 -b 0.0.0.0:5005 --log-level debug app:app diff --git a/sass/base_theme.scss b/sass/base_theme.scss index c4c67b2..116f259 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -270,6 +270,15 @@ a:hover { float: left; border-radius:2px; } +.bar-icon { + background: $background-color; + padding: 5px; + color: $color-light; + margin-right: 10px; + float: left; + border: none; +} + .bar-item:hover { background: $primary-color; color: $background-color; diff --git a/templates/layout.html b/templates/layout.html index 08a7761..e97787d 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -6,6 +6,7 @@ {% block title %}{{ config.NAME }}{% endblock %}'s microblog + {% if not request.args.get("older_than") and not request.args.get("previous_than") %}{% endif %} diff --git a/templates/utils.html b/templates/utils.html index a63895e..3a1fa6c 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -56,6 +56,7 @@ {% if not perma %} + @@ -164,10 +165,14 @@ {% if perma %} {{ obj.published | format_time }} {% if not (obj.id | is_from_outbox) %} -permalink + + + {% endif %} {% else %} -permalink + + + {% endif %} {% if meta.count_reply and obj.id | is_from_outbox %}{{ meta.count_reply }} replies @@ -192,7 +197,9 @@ - + {% endif %} diff --git a/utils/migrations.py b/utils/migrations.py new file mode 100644 index 0000000..cbf6eb2 --- /dev/null +++ b/utils/migrations.py @@ -0,0 +1,56 @@ +"""Automatic migration tools for the da:ta stored in MongoDB.""" +import logging +from abc import ABC +from abc import abstractmethod +from typing import List +from typing import Type + +from config import DB + +logger = logging.getLogger(__name__) + +# Used to keep track of all the defined migrations +_MIGRATIONS: List[Type["Migration"]] = [] + + +def perform() -> None: + """Perform all the defined migration.""" + for migration in _MIGRATIONS: + migration().perform() + + +class Migration(ABC): + """Abstract class for migrations.""" + + def __init__(self) -> None: + self.name = self.__class__.__qualname__ + self._col = DB.migrations + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + _MIGRATIONS.append(cls) + + def _apply(self) -> None: + self._col.insert_one({"name": self.name}) + + def _reset(self) -> None: + self._col.delete_one({"name": self.name}) + + def _is_applied(self) -> bool: + return bool(self._col.find_one({"name": self.name})) + + @abstractmethod + def migrate(self) -> None: + """Expected to be implemented by actual migrations.""" + pass + + def perform(self) -> None: + if self._is_applied(): + logger.info(f"Skipping migration {self.name} (already applied)") + return + + logger.info(f"Performing migration {self.name}...") + self.migrate() + + self._apply() + logger.info("Done") From 866979309c7021f182d97fafd6d3501f6ba00171 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 14 Jul 2019 23:58:10 +0200 Subject: [PATCH 0506/1425] Formatting --- migrations.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/migrations.py b/migrations.py index 1ec997f..e9ef95a 100644 --- a/migrations.py +++ b/migrations.py @@ -60,11 +60,15 @@ class _1_MetaMigrationt(Migration): # Set `meta.object_visibility` (str) if not data["meta"].get("object_visibility"): set_meta["meta.object_visibility"] = None - object_id = data["meta"].get("object_id") or set_meta.get("meta.object_id") + object_id = data["meta"].get("object_id") or set_meta.get( + "meta.object_id" + ) if object_id: obj = data["meta"].get("object") or data["activity"].get("object") if isinstance(obj, dict): - set_meta["meta.object_visibility"] = self.__guess_visibility(obj).name + set_meta["meta.object_visibility"] = self.__guess_visibility( + obj + ).name # Set `meta.actor_id` (str) if not data["meta"].get("actor_id"): @@ -92,7 +96,9 @@ class _1_MetaMigrationt(Migration): # Set `meta.visibility` (str) if not data["meta"].get("visibility"): - set_meta["meta.visibility"] = self.__guess_visibility(data["activity"]).name + set_meta["meta.visibility"] = self.__guess_visibility( + data["activity"] + ).name if not data["meta"].get("server"): set_meta["meta.server"] = urlparse(data["remote_id"]).netloc From 8e977a2b149928e3fa01ced6ad7c388464df6c2a Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 15 Jul 2019 23:08:12 +0200 Subject: [PATCH 0507/1425] More work on post visibility --- activitypub.py | 8 ++++++-- app.py | 17 +++++++++++++++++ migrations.py | 3 ++- sass/base_theme.scss | 19 ++++++++++--------- templates/index.html | 2 +- templates/layout.html | 2 +- templates/liked.html | 2 +- templates/stream.html | 6 +++--- templates/utils.html | 38 ++++++++++++++++++-------------------- 9 files changed, 59 insertions(+), 38 deletions(-) diff --git a/activitypub.py b/activitypub.py index 82a259d..b49b12a 100644 --- a/activitypub.py +++ b/activitypub.py @@ -124,6 +124,9 @@ class MicroblogPubBackend(Backend): object_id = activity.get_object_id() except ValueError: pass + object_visibility = None + if object_id: + object_visibility = ap.get_visibility(activity.get_object()) actor_id = activity.get_actor().id @@ -141,6 +144,7 @@ class MicroblogPubBackend(Backend): "visibility": visibility.name, "actor_id": actor_id, "object_id": object_id, + "object_visibility": object_visibility.name, "poll_answer": False, }, } @@ -670,7 +674,7 @@ def gen_feed(): fg.logo(ME.get("icon", {}).get("url")) fg.language("en") for item in DB.activities.find( - {"box": Box.OUTBOX.value, "type": "Create", "meta.deleted": False}, limit=10 + {"box": Box.OUTBOX.value, "type": "Create", "meta.deleted": False, "meta.public": True}, limit=10 ).sort("_id", -1): fe = fg.add_entry() fe.id(item["activity"]["object"].get("url")) @@ -684,7 +688,7 @@ def json_feed(path: str) -> Dict[str, Any]: """JSON Feed (https://jsonfeed.org/) document.""" data = [] for item in DB.activities.find( - {"box": Box.OUTBOX.value, "type": "Create", "meta.deleted": False}, limit=10 + {"box": Box.OUTBOX.value, "type": "Create", "meta.deleted": False, "meta.public": True}, limit=10 ).sort("_id", -1): data.append( { diff --git a/app.py b/app.py index 60ea41a..1116054 100644 --- a/app.py +++ b/app.py @@ -247,6 +247,16 @@ def _get_file_url(url, size, kind): return url +@app.template_filter() +def visibility(v: str) -> str: + return ap.Visibility[v].value.lower() + + +@app.template_filter() +def visibility_is_public(v: str) -> bool: + return v in [ap.Visibility.PUBLIC.name, ap.Visibility.UNLISTED.name] + + @app.template_filter() def emojify(text): return emoji_unicode.replace( @@ -1691,6 +1701,10 @@ def api_delete(): def api_boost(): note = _user_api_get_note() + # Ensures the note visibility allow us to build an Announce (in respect to the post visibility) + if ap.get_visibility(note) not in [ap.Visibility.PUBLIC, ap.VISIBILITY.UNLISTED]: + abort(400) + announce = note.build_announce(MY_PERSON) announce_id = post_to_outbox(announce) @@ -1841,6 +1855,7 @@ def admin_bookmarks(): @app.route("/inbox", methods=["GET", "POST"]) # noqa: C901 def inbox(): + # GET /inbox if request.method == "GET": if not is_api_request(): abort(404) @@ -1859,6 +1874,7 @@ def inbox(): ) ) + # POST/ inbox try: data = request.get_json(force=True) except Exception: @@ -3125,6 +3141,7 @@ def task_fetch_remote_question(): } ) remote_question = get_backend().fetch_iri(iri, no_cache=True) + # FIXME(tsileo): compute and set `meta.object_visiblity` (also update utils.py to do it) if ( local_question and ( diff --git a/migrations.py b/migrations.py index e9ef95a..4ce230c 100644 --- a/migrations.py +++ b/migrations.py @@ -104,4 +104,5 @@ class _1_MetaMigrationt(Migration): set_meta["meta.server"] = urlparse(data["remote_id"]).netloc logger.info(f"meta={set_meta}\n") - DB.activities.update_one({"_id": data["_id"]}, {"$set": set_meta}) + if set_meta: + DB.activities.update_one({"_id": data["_id"]}, {"$set": set_meta}) diff --git a/sass/base_theme.scss b/sass/base_theme.scss index 116f259..8b77b56 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -270,19 +270,20 @@ a:hover { float: left; border-radius:2px; } -.bar-icon { - background: $background-color; - padding: 5px; - color: $color-light; - margin-right: 10px; - float: left; - border: none; -} - .bar-item:hover { background: $primary-color; color: $background-color; } +.bar-item-no-border { + color: $color-light; + background: inherit; + cursor: default; +} +.bar-item-no-border:hover { + color: $color-light; + background: inherit; + cursor: default; +} button.bar-item { border: 0 } diff --git a/templates/index.html b/templates/index.html index f1c9711..527918d 100644 --- a/templates/index.html +++ b/templates/index.html @@ -53,7 +53,7 @@ {% endif %} {% endif %} {% if item.meta.object %} - {{ utils.display_note(item.meta.object, ui=False, meta={'actor': item.meta.object_actor}) }} + {{ utils.display_note(item.meta.object, ui=False, meta=item.meta) }} {% endif %} {% elif item | has_type('Create') %} {{ utils.display_note(item.activity.object, meta=item.meta, no_color=True) }} diff --git a/templates/layout.html b/templates/layout.html index e97787d..65ff664 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -6,13 +6,13 @@ {% block title %}{{ config.NAME }}{% endblock %}'s microblog - {% if not request.args.get("older_than") and not request.args.get("previous_than") %}{% endif %} {% block links %}{% endblock %} {% if config.THEME_COLOR %}{% endif %} {% block headers %}{% endblock %} diff --git a/templates/new.html b/templates/new.html index ce85c02..fb5a26b 100644 --- a/templates/new.html +++ b/templates/new.html @@ -32,6 +32,9 @@ {% for emoji in emojis %} {{ emoji | emojify | safe }} {% endfor %} +{% for emoji in custom_emojis.values() %} +{{emoji.name}} +{% endfor %}

      diff --git a/templates/utils.html b/templates/utils.html index d6d298a..5cc293d 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -8,7 +8,7 @@ {% endif %}
      -
      {{ follower.name or follower.preferredUsername }}
      +
      {{ (follower.name or follower.preferredUsername) | clean | replace_custom_emojis(follower) | safe }}
      @{{ follower.preferredUsername }}@{{ follower | url_or_id | get_url | domain }}
      @@ -53,8 +53,9 @@
      - {{ actor.name or actor.preferredUsername }} - @{% if not no_color and obj.id | is_from_outbox %}{{ actor.preferredUsername }}{% else %}{{ actor.preferredUsername }}{% endif %}@{% if not no_color and obj.id | is_from_outbox %}{{ actor | url_or_id | get_url | domain }}{% else %}{{ actor | url_or_id | get_url | domain }}{% endif %} + + {{ (actor.name or actor.preferredUsername) | clean | replace_custom_emojis(actor) | safe }} + @{% if not no_color and obj.id | is_from_outbox %}{{ actor.preferredUsername | clean | replace_custom_emojis(actor) | safe }}{% else %}{{ actor.preferredUsername | clean | replace_custom_emojis(actor) | safe }}{% endif %}@{% if not no_color and obj.id | is_from_outbox %}{{ actor | url_or_id | get_url | domain }}{% else %}{{ actor | url_or_id | get_url | domain }}{% endif %} {% if not perma %} @@ -64,7 +65,7 @@ {% endif %}
      - {% if obj.summary %}

      {{ obj.summary | clean | safe }}

      {% endif %} + {% if obj.summary %}

      {{ obj.summary | clean | replace_custom_emojis(obj) | safe }}

      {% endif %} {% if obj | has_type('Video') %}
      diff --git a/utils/emojis.py b/utils/emojis.py new file mode 100644 index 0000000..f0c9539 --- /dev/null +++ b/utils/emojis.py @@ -0,0 +1,47 @@ +import mimetypes +import re +from datetime import datetime +from pathlib import Path +from typing import Any +from typing import Dict +from typing import List +from typing import Set + +from little_boxes import activitypub as ap + +EMOJI_REGEX = re.compile(r"(:[\d\w]+:)") + +EMOJIS: Dict[str, ap.Emoji] = {} +EMOJIS_BY_NAME: Dict[str, ap.Emoji] = {} + + +def _load_emojis(root_dir: Path, base_url: str) -> None: + if EMOJIS: + return + for emoji in (root_dir / "static" / "emojis").iterdir(): + mt = mimetypes.guess_type(emoji.name)[0] + if mt and mt.startswith("image/"): + name = emoji.name.split(".")[0] + ap_emoji = ap.Emoji( + name=f":{name}:", + updated=ap.format_datetime(datetime.fromtimestamp(0.0).astimezone()), + id=f"{base_url}/emoji/{name}", + icon={ + "mediaType": mt, + "type": ap.ActivityType.IMAGE.value, + "url": f"{base_url}/static/emojis/{emoji.name}", + }, + ) + EMOJIS[emoji.name] = ap_emoji + EMOJIS_BY_NAME[ap_emoji.name] = ap_emoji + + +def tags(content: str) -> List[Dict[str, Any]]: + tags: List[Dict[str, Any]] = [] + added: Set[str] = set() + for e in re.findall(EMOJI_REGEX, content): + if e not in added and e in EMOJIS_BY_NAME: + tags.append(EMOJIS_BY_NAME[e].to_dict()) + added.add(e) + + return tags diff --git a/utils/media.py b/utils/media.py index 96567de..9b0cb81 100644 --- a/utils/media.py +++ b/utils/media.py @@ -5,8 +5,11 @@ from enum import unique from functools import lru_cache from gzip import GzipFile from io import BytesIO +from shutil import copyfileobj from typing import Any from typing import Dict +from typing import Optional +from typing import Tuple import gridfs import piexif @@ -31,13 +34,26 @@ def is_video(filename): return False -def load(url: str, user_agent: str) -> Image: +def _load(url: str, user_agent: str) -> Tuple[BytesIO, Optional[str]]: """Initializes a `PIL.Image` from the URL.""" + out = BytesIO() with requests.get(url, stream=True, headers={"User-Agent": user_agent}) as resp: resp.raise_for_status() resp.raw.decode_content = True - return Image.open(BytesIO(resp.raw.read())) + while 1: + buf = resp.raw.read() + if not buf: + break + out.write(buf) + out.seek(0) + return out, resp.headers.get("content-type") + + +def load(url: str, user_agent: str) -> Image: + """Initializes a `PIL.Image` from the URL.""" + out, _ = _load(url, user_agent) + return Image.open(out) def to_data_uri(img: Image) -> str: @@ -54,6 +70,7 @@ class Kind(Enum): ACTOR_ICON = "actor_icon" UPLOAD = "upload" OG_IMAGE = "og" + EMOJI = "emoji" class MediaCache(object): @@ -173,6 +190,26 @@ class MediaCache(object): kind=Kind.ACTOR_ICON.value, ) + def is_emoji_cached(self, url: str) -> bool: + return bool(self.fs.find_one({"url": url, "kind": Kind.EMOJI.value})) + + def cache_emoji(self, url: str, iri: str) -> None: + if self.is_emoji_cached(url): + return + src, content_type = _load(url, self.user_agent) + with BytesIO() as buf: + with GzipFile(mode="wb", fileobj=buf) as g: + copyfileobj(src, g) + buf.seek(0) + self.fs.put( + buf, + url=url, + remote_id=iri, + size=None, + content_type=content_type or mimetypes.guess_type(url)[0], + kind=Kind.EMOJI.value, + ) + def save_upload(self, obuf: BytesIO, filename: str) -> str: # Remove EXIF metadata if filename.lower().endswith(".jpg") or filename.lower().endswith(".jpeg"): diff --git a/utils/template_filters.py b/utils/template_filters.py index e565b15..93f3b88 100644 --- a/utils/template_filters.py +++ b/utils/template_filters.py @@ -91,6 +91,25 @@ ALLOWED_TAGS = [ ] +@filters.app_template_filter() +def replace_custom_emojis(content, note): + print("\n" * 50) + print("custom_replace", note) + idx = {} + for tag in note.get("tag", []): + if tag.get("type") == "Emoji": + # try: + idx[tag["name"]] = _get_file_url(tag["icon"]["url"], None, Kind.EMOJI) + + for emoji_name, emoji_url in idx.items(): + content = content.replace( + emoji_name, + f'{emoji_name}', + ) + + return content + + def clean_html(html): try: return bleach.clean(html, tags=ALLOWED_TAGS, strip=True) @@ -237,6 +256,9 @@ _FILE_URL_CACHE = LRUCache(4096) def _get_file_url(url, size, kind) -> str: + if url.startswith(BASE_URL): + return url + k = (url, size, kind) cached = _FILE_URL_CACHE.get(k) if cached: @@ -249,8 +271,6 @@ def _get_file_url(url, size, kind) -> str: return out _logger.error(f"cache not available for {url}/{size}/{kind}") - if url.startswith(BASE_URL): - return url p = urlparse(url) return f"/p/{p.scheme}" + p._replace(scheme="").geturl()[1:] From 2cd0835208197247fd46356fd3a1e2ebbc82b1b7 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 20 Aug 2019 22:55:55 +0200 Subject: [PATCH 0637/1425] Fix the media proxy --- app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index a41bcf2..e2d935d 100644 --- a/app.py +++ b/app.py @@ -258,7 +258,7 @@ def proxy(scheme: str, url: str) -> Any: } req_headers["Host"] = urlparse(url).netloc resp = requests.get(url, stream=True, headers=req_headers) - app.logger.info(f"proxied req {url}: {resp!r}") + app.logger.info(f"proxied req {url} {req_headers}: {resp!r}") def data(): for chunk in resp.raw.stream(decode_content=False): @@ -278,7 +278,7 @@ def proxy(scheme: str, url: str) -> Any: "last-modified", ] } - return Response(data(), headers=resp_headers) + return Response(data(), headers=resp_headers, status=resp.status_code) @app.route("/media/") From 6d0d1d64801e9c5ccc7a8df1d036a9d2cd9682d8 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 20 Aug 2019 23:31:37 +0200 Subject: [PATCH 0638/1425] Fix for buggy updates --- core/activitypub.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/core/activitypub.py b/core/activitypub.py index 93a25f6..e43653c 100644 --- a/core/activitypub.py +++ b/core/activitypub.py @@ -179,7 +179,16 @@ def post_to_inbox(activity: ap.BaseActivity) -> None: Tasks.process_reply(activity.get_object_id()) return - # Hubzilla forward activities in a Create, process them as possible replies + # Hubzilla sends Update with the same ID as the actor, and it poisons the cache + if ( + activity.has_type(ap.ActivityType.UPDATE) + and activity.id == activity.get_object_id() + ): + # Start a task to update the cached actor + Tasks.cache_actor(activity.id) + return + + # Hubzilla forwards activities in a Create, process them as possible replies if activity.has_type(ap.ActivityType.CREATE) and server(activity.id) != server( activity.get_object_id() ): From 6592841558be714598052d3c9fbca4dfe320a74f Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 21 Aug 2019 00:28:24 +0200 Subject: [PATCH 0639/1425] Fix emoji caching --- utils/media.py | 15 +++++++++------ utils/template_filters.py | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/utils/media.py b/utils/media.py index 9b0cb81..789c668 100644 --- a/utils/media.py +++ b/utils/media.py @@ -196,17 +196,20 @@ class MediaCache(object): def cache_emoji(self, url: str, iri: str) -> None: if self.is_emoji_cached(url): return - src, content_type = _load(url, self.user_agent) - with BytesIO() as buf: - with GzipFile(mode="wb", fileobj=buf) as g: - copyfileobj(src, g) + i = load(url, self.user_agent) + for size in [25]: + t1 = i.copy() + t1.thumbnail((size, size)) + with BytesIO() as buf: + with GzipFile(mode="wb", fileobj=buf) as f1: + t1.save(f1, format=i.format) buf.seek(0) self.fs.put( buf, url=url, + size=size, remote_id=iri, - size=None, - content_type=content_type or mimetypes.guess_type(url)[0], + content_type=i.get_format_mimetype(), kind=Kind.EMOJI.value, ) diff --git a/utils/template_filters.py b/utils/template_filters.py index 93f3b88..11987d1 100644 --- a/utils/template_filters.py +++ b/utils/template_filters.py @@ -99,7 +99,7 @@ def replace_custom_emojis(content, note): for tag in note.get("tag", []): if tag.get("type") == "Emoji": # try: - idx[tag["name"]] = _get_file_url(tag["icon"]["url"], None, Kind.EMOJI) + idx[tag["name"]] = _get_file_url(tag["icon"]["url"], 25, Kind.EMOJI) for emoji_name, emoji_url in idx.items(): content = content.replace( From 40c7f4c7f0c530e7b6fba406bcefd4295ed1bbb0 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 21 Aug 2019 07:40:26 +0200 Subject: [PATCH 0640/1425] Remove unused import --- utils/media.py | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/media.py b/utils/media.py index 789c668..19288df 100644 --- a/utils/media.py +++ b/utils/media.py @@ -5,7 +5,6 @@ from enum import unique from functools import lru_cache from gzip import GzipFile from io import BytesIO -from shutil import copyfileobj from typing import Any from typing import Dict from typing import Optional From 5dd6385fe6ce9a37d95bcc41a62c48c24c135c67 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 22 Aug 2019 00:45:49 +0200 Subject: [PATCH 0641/1425] Fix the published flag for outbox activities --- blueprints/tasks.py | 10 ++-------- core/activitypub.py | 21 ++++++++++++--------- core/meta.py | 4 ++++ 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/blueprints/tasks.py b/blueprints/tasks.py index 880bf0e..d2545dc 100644 --- a/blueprints/tasks.py +++ b/blueprints/tasks.py @@ -32,9 +32,9 @@ from core.meta import MetaKey from core.meta import by_object_id from core.meta import by_remote_id from core.meta import by_type -from core.meta import flag from core.meta import inc from core.meta import upsert +from core.notifications import _NewMeta from core.notifications import set_inbox_flags from core.outbox import process_outbox from core.remote import track_failed_send @@ -44,7 +44,6 @@ from core.shared import _Response from core.shared import back from core.shared import p from core.tasks import Tasks -from utils import now from utils import opengraph from utils.media import is_video @@ -643,12 +642,7 @@ def task_process_new_activity() -> _Response: activity = ap.fetch_remote_activity(iri) app.logger.info(f"activity={activity!r}") - flags = {} - - if not activity.published: - flags.update(flag(MetaKey.PUBLISHED, now())) - else: - flags.update(flag(MetaKey.PUBLISHED, activity.published)) + flags: _NewMeta = {} set_inbox_flags(activity, flags) app.logger.info(f"a={activity}, flags={flags!r}") diff --git a/core/activitypub.py b/core/activitypub.py index e43653c..2559f71 100644 --- a/core/activitypub.py +++ b/core/activitypub.py @@ -123,15 +123,18 @@ def save(box: Box, activity: ap.BaseActivity) -> None: "type": _to_list(activity.type), "remote_id": activity.id, "meta": { - "undo": False, - "deleted": False, - "public": is_public, - "server": urlparse(activity.id).netloc, - "visibility": visibility.name, - "actor_id": actor_id, - "object_id": object_id, - "object_visibility": object_visibility, - "poll_answer": False, + MetaKey.UNDO.value: False, + MetaKey.DELETED.value: False, + MetaKey.PUBLIC.value: is_public, + MetaKey.SERVER.value: urlparse(activity.id).netloc, + MetaKey.VISIBILITY.value: visibility.name, + MetaKey.ACTOR_ID.value: actor_id, + MetaKey.OBJECT_ID.value: object_id, + MetaKey.OBJECT_VISIBILITY.value: object_visibility, + MetaKey.POLL_ANSWER.value: False, + MetaKey.PUBLISHED.value: activity.published + if activity.published + else now(), }, } ) diff --git a/core/meta.py b/core/meta.py index 2b51346..6a6ccef 100644 --- a/core/meta.py +++ b/core/meta.py @@ -39,6 +39,10 @@ class MetaKey(Enum): PUBLIC = "public" THREAD_ROOT_PARENT = "thread_root_parent" + SERVER = "server" + VISIBILITY = "visibility" + OBJECT_VISIBILITY = "object_visibility" + DELETED = "deleted" BOOSTED = "boosted" LIKED = "liked" From 972e1293816549fc430c080bc2758cfd2d741c18 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 23 Aug 2019 23:49:33 +0200 Subject: [PATCH 0642/1425] Support gzip compression for JSON resp --- core/shared.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/core/shared.py b/core/shared.py index 5d24198..d2d212f 100644 --- a/core/shared.py +++ b/core/shared.py @@ -1,7 +1,10 @@ +import gzip import json import os from functools import wraps from typing import Any +from typing import Dict +from typing import Tuple import flask from bson.objectid import ObjectId @@ -43,12 +46,27 @@ ap.use_backend(back) MY_PERSON = ap.Person(**ME) +def build_resp(resp): + """Encode the response to gzip if supported by the client.""" + headers = {} + accept_encoding = request.headers.get("Accept-Encoding", "") + if "gzip" in accept_encoding.lower(): + return ( + gzip.compress(resp.encode(), compresslevel=6), + {"Vary": "Accept-Encoding", "Content-Encoding": "gzip"}, + ) + + return resp, headers + + def jsonify(**data): if "@context" not in data: data["@context"] = config.DEFAULT_CTX + resp, headers = build_resp(json.dumps(data)) return Response( - response=json.dumps(data), + response=resp, headers={ + **headers, "Cache-Control": "max-age=0, private, must-revalidate", "Content-Type": "application/json" if app.debug From 9432128540cbcec15673f4e2cff9afae45294a79 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 24 Aug 2019 00:03:26 +0200 Subject: [PATCH 0643/1425] Add support for audio attachment --- templates/utils.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/templates/utils.html b/templates/utils.html index 5cc293d..dddc42c 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -185,6 +185,8 @@ {% elif (a.mediaType and a.mediaType.startswith("video/")) %}
      • + {% elif (a.mediaType and a.mediaType.startswith("audio/")) %} +
      • {% else %}
    • {% if a.filename %}{{ a.filename }}{% else %}{{ a.url }}{% endif %}
      • {% endif %} @@ -320,7 +322,8 @@ -{% endif %} +{% endif %} {% endif %} From 908f53ef4ba8b28dddd72bf8bc5983805f024bc9 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 24 Aug 2019 00:04:56 +0200 Subject: [PATCH 0644/1425] Remove unused imports --- core/shared.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/shared.py b/core/shared.py index d2d212f..6dc7125 100644 --- a/core/shared.py +++ b/core/shared.py @@ -3,8 +3,6 @@ import json import os from functools import wraps from typing import Any -from typing import Dict -from typing import Tuple import flask from bson.objectid import ObjectId From d56d5f5a200fc146fa3fd1973a877df1a9079a77 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 24 Aug 2019 00:19:59 +0200 Subject: [PATCH 0645/1425] Tweak attachments display --- templates/utils.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/utils.html b/templates/utils.html index dddc42c..caeace5 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -178,7 +178,7 @@ {% if obj.attachment | not_only_imgs %}

      Attachments

      -
        +
          {% endif %} {% for a in obj.attachment %} {% if (a.mediaType and a.mediaType.startswith("image/")) or (a.type and a.type == 'Image') %} @@ -186,7 +186,7 @@ {% elif (a.mediaType and a.mediaType.startswith("video/")) %}
          • {% elif (a.mediaType and a.mediaType.startswith("audio/")) %} -
          • +
          • {% else %}
        • {% if a.filename %}{{ a.filename }}{% else %}{{ a.url }}{% endif %}
          • {% endif %} From 784362b5321377b30d001680df71c006754b1871 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 24 Aug 2019 00:24:57 +0200 Subject: [PATCH 0646/1425] Add caching for gzipped response --- core/shared.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/shared.py b/core/shared.py index 6dc7125..5ea6db9 100644 --- a/core/shared.py +++ b/core/shared.py @@ -1,6 +1,7 @@ import gzip import json import os +from functools import lru_cache from functools import wraps from typing import Any @@ -44,6 +45,7 @@ ap.use_backend(back) MY_PERSON = ap.Person(**ME) +@lru_cache(512) def build_resp(resp): """Encode the response to gzip if supported by the client.""" headers = {} From dad3dae988003162aa39cf241f2b87cb00a23d7e Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 24 Aug 2019 10:41:31 +0200 Subject: [PATCH 0647/1425] Tweak AP ctx, and gzip support for HTML resp --- app.py | 131 +++++++++++++++++++++----------------- blueprints/admin.py | 137 +++++++++++++++++++++++----------------- blueprints/indieauth.py | 21 +++--- blueprints/tasks.py | 4 ++ config.py | 2 +- core/activitypub.py | 3 + core/shared.py | 18 ++++-- 7 files changed, 188 insertions(+), 128 deletions(-) diff --git a/app.py b/app.py index e2d935d..b0cb623 100644 --- a/app.py +++ b/app.py @@ -63,9 +63,10 @@ from core.meta import is_public from core.meta import not_undo from core.shared import _build_thread from core.shared import _get_ip +from core.shared import activitypubify from core.shared import csrf +from core.shared import htmlify from core.shared import is_api_request -from core.shared import jsonify from core.shared import login_required from core.shared import noindex from core.shared import paginated_query @@ -322,7 +323,7 @@ def serve_uploads(oid, fname): def remote_follow(): """Form to allow visitor to perform the remote follow dance.""" if request.method == "GET": - return render_template("remote_follow.html") + return htmlify(render_template("remote_follow.html")) csrf.protect() profile = request.form.get("profile") @@ -339,8 +340,7 @@ def remote_follow(): def index(): if is_api_request(): _log_sig() - print(ME) - return jsonify(**ME) + return activitypubify(**ME) q = { "box": Box.OUTBOX.value, @@ -369,14 +369,15 @@ def index(): DB.activities, q, limit=25 - len(pinned) ) - resp = render_template( - "index.html", - outbox_data=outbox_data, - older_than=older_than, - newer_than=newer_than, - pinned=pinned, + return htmlify( + render_template( + "index.html", + outbox_data=outbox_data, + older_than=older_than, + newer_than=newer_than, + pinned=pinned, + ) ) - return resp @app.route("/all") @@ -391,11 +392,13 @@ def all(): } outbox_data, older_than, newer_than = paginated_query(DB.activities, q) - return render_template( - "index.html", - outbox_data=outbox_data, - older_than=older_than, - newer_than=newer_than, + return htmlify( + render_template( + "index.html", + outbox_data=outbox_data, + older_than=older_than, + newer_than=newer_than, + ) ) @@ -458,8 +461,10 @@ def note_by_id(note_id): app.logger.exception(f"invalid doc: {doc!r}") app.logger.info(f"shares={shares!r}") - return render_template( - "note.html", likes=likes, shares=shares, thread=thread, note=data + return htmlify( + render_template( + "note.html", likes=likes, shares=shares, thread=thread, note=data + ) ) @@ -477,7 +482,7 @@ def outbox(): "meta.public": True, "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, } - return jsonify( + return activitypubify( **activitypub.build_ordered_collection( DB.activities, q=q, @@ -503,7 +508,9 @@ def outbox(): @app.route("/emoji/") def ap_emoji(name): if name in EMOJIS: - return jsonify(**{**EMOJIS[name].to_dict(), "@context": config.DEFAULT_CTX}) + return activitypubify( + **{**EMOJIS[name].to_dict(), "@context": config.DEFAULT_CTX} + ) abort(404) @@ -523,7 +530,7 @@ def outbox_detail(item_id): if doc["meta"].get("deleted", False): abort(404) - return jsonify(**activity_from_doc(doc)) + return activitypubify(**activity_from_doc(doc)) @app.route("/outbox//activity") @@ -541,7 +548,7 @@ def outbox_activity(item_id): if obj["type"] != ActivityType.CREATE.value: abort(404) - return jsonify(**obj["object"]) + return activitypubify(**obj["object"]) @app.route("/outbox//replies") @@ -570,7 +577,7 @@ def outbox_activity_replies(item_id): "activity.object.inReplyTo": obj.get_object().id, } - return jsonify( + return activitypubify( **activitypub.build_ordered_collection( DB.activities, q=q, @@ -610,7 +617,7 @@ def outbox_activity_likes(item_id): ], } - return jsonify( + return activitypubify( **activitypub.build_ordered_collection( DB.activities, q=q, @@ -649,7 +656,7 @@ def outbox_activity_shares(item_id): ], } - return jsonify( + return activitypubify( **activitypub.build_ordered_collection( DB.activities, q=q, @@ -672,7 +679,7 @@ def inbox(): except BadSignature: abort(404) - return jsonify( + return activitypubify( **activitypub.build_ordered_collection( DB.activities, q={"meta.deleted": False, "box": Box.INBOX.value}, @@ -800,7 +807,7 @@ def followers(): if is_api_request(): _log_sig() - return jsonify( + return activitypubify( **activitypub.build_ordered_collection( DB.activities, q=q, @@ -814,11 +821,13 @@ def followers(): followers = [ doc["meta"]["actor"] for doc in raw_followers if "actor" in doc.get("meta", {}) ] - return render_template( - "followers.html", - followers_data=followers, - older_than=older_than, - newer_than=newer_than, + return htmlify( + render_template( + "followers.html", + followers_data=followers, + older_than=older_than, + newer_than=newer_than, + ) ) @@ -829,11 +838,11 @@ def following(): if is_api_request(): _log_sig() if config.HIDE_FOLLOWING: - return jsonify( + return activitypubify( **activitypub.simple_build_ordered_collection("following", []) ) - return jsonify( + return activitypubify( **activitypub.build_ordered_collection( DB.activities, q=q, @@ -853,12 +862,14 @@ def following(): if "remote_id" in doc and "object" in doc.get("meta", {}) ] lists = list(DB.lists.find()) - return render_template( - "following.html", - following_data=following, - older_than=older_than, - newer_than=newer_than, - lists=lists, + return htmlify( + render_template( + "following.html", + following_data=following, + older_than=older_than, + newer_than=newer_than, + lists=lists, + ) ) @@ -873,18 +884,20 @@ def tags(tag): ): abort(404) if not is_api_request(): - return render_template( - "tags.html", - tag=tag, - outbox_data=DB.activities.find( - { - "box": Box.OUTBOX.value, - "type": ActivityType.CREATE.value, - "meta.deleted": False, - "activity.object.tag.type": "Hashtag", - "activity.object.tag.name": "#" + tag, - } - ), + return htmlify( + render_template( + "tags.html", + tag=tag, + outbox_data=DB.activities.find( + { + "box": Box.OUTBOX.value, + "type": ActivityType.CREATE.value, + "meta.deleted": False, + "activity.object.tag.type": "Hashtag", + "activity.object.tag.name": "#" + tag, + } + ), + ) ) _log_sig() q = { @@ -895,7 +908,7 @@ def tags(tag): "activity.object.tag.type": "Hashtag", "activity.object.tag.name": "#" + tag, } - return jsonify( + return activitypubify( **activitypub.build_ordered_collection( DB.activities, q=q, @@ -920,7 +933,9 @@ def featured(): "meta.pinned": True, } data = [clean_activity(doc["activity"]["object"]) for doc in DB.activities.find(q)] - return jsonify(**activitypub.simple_build_ordered_collection("featured", data)) + return activitypubify( + **activitypub.simple_build_ordered_collection("featured", data) + ) @app.route("/liked") @@ -936,12 +951,14 @@ def liked(): liked, older_than, newer_than = paginated_query(DB.activities, q) - return render_template( - "liked.html", liked=liked, older_than=older_than, newer_than=newer_than + return htmlify( + render_template( + "liked.html", liked=liked, older_than=older_than, newer_than=newer_than + ) ) q = {"meta.deleted": False, "meta.undo": False, "type": ActivityType.LIKE.value} - return jsonify( + return activitypubify( **activitypub.build_ordered_collection( DB.activities, q=q, diff --git a/blueprints/admin.py b/blueprints/admin.py index ec87b6e..3bae40e 100644 --- a/blueprints/admin.py +++ b/blueprints/admin.py @@ -32,6 +32,7 @@ from core.shared import MY_PERSON from core.shared import _build_thread from core.shared import _Response from core.shared import csrf +from core.shared import htmlify from core.shared import login_required from core.shared import noindex from core.shared import p @@ -113,7 +114,9 @@ def admin_login() -> _Response: payload = u2f.begin_authentication(ID, devices) session["challenge"] = payload - return render_template("login.html", u2f_enabled=u2f_enabled, payload=payload) + return htmlify( + render_template("login.html", u2f_enabled=u2f_enabled, payload=payload) + ) @blueprint.route("/admin", methods=["GET"]) @@ -127,47 +130,53 @@ def admin_index() -> _Response: } col_liked = DB.activities.count(q) - return render_template( - "admin.html", - instances=list(DB.instances.find()), - inbox_size=DB.activities.count({"box": Box.INBOX.value}), - outbox_size=DB.activities.count({"box": Box.OUTBOX.value}), - col_liked=col_liked, - col_followers=DB.activities.count( - { - "box": Box.INBOX.value, - "type": ap.ActivityType.FOLLOW.value, - "meta.undo": False, - } - ), - col_following=DB.activities.count( - { - "box": Box.OUTBOX.value, - "type": ap.ActivityType.FOLLOW.value, - "meta.undo": False, - } - ), + return htmlify( + render_template( + "admin.html", + instances=list(DB.instances.find()), + inbox_size=DB.activities.count({"box": Box.INBOX.value}), + outbox_size=DB.activities.count({"box": Box.OUTBOX.value}), + col_liked=col_liked, + col_followers=DB.activities.count( + { + "box": Box.INBOX.value, + "type": ap.ActivityType.FOLLOW.value, + "meta.undo": False, + } + ), + col_following=DB.activities.count( + { + "box": Box.OUTBOX.value, + "type": ap.ActivityType.FOLLOW.value, + "meta.undo": False, + } + ), + ) ) @blueprint.route("/admin/indieauth", methods=["GET"]) @login_required def admin_indieauth() -> _Response: - return render_template( - "admin_indieauth.html", - indieauth_actions=DB.indieauth.find().sort("ts", -1).limit(100), + return htmlify( + render_template( + "admin_indieauth.html", + indieauth_actions=DB.indieauth.find().sort("ts", -1).limit(100), + ) ) @blueprint.route("/admin/tasks", methods=["GET"]) @login_required def admin_tasks() -> _Response: - return render_template( - "admin_tasks.html", - success=p.get_success(), - dead=p.get_dead(), - waiting=p.get_waiting(), - cron=p.get_cron(), + return htmlify( + render_template( + "admin_tasks.html", + success=p.get_success(), + dead=p.get_dead(), + waiting=p.get_waiting(), + cron=p.get_cron(), + ) ) @@ -191,8 +200,10 @@ def admin_lookup() -> _Response: print(data) app.logger.debug(data.to_dict()) - return render_template( - "lookup.html", data=data, meta=meta, url=request.args.get("url") + return htmlify( + render_template( + "lookup.html", data=data, meta=meta, url=request.args.get("url") + ) ) @@ -214,7 +225,7 @@ def admin_thread() -> _Response: tpl = "note.html" if request.args.get("debug"): tpl = "note_debug.html" - return render_template(tpl, thread=thread, note=data) + return htmlify(render_template(tpl, thread=thread, note=data)) @blueprint.route("/admin/new", methods=["GET"]) @@ -246,14 +257,16 @@ def admin_new() -> _Response: content = f"@{actor.preferredUsername}@{domain} " thread = _build_thread(data) - return render_template( - "new.html", - reply=reply_id, - content=content, - thread=thread, - visibility=ap.Visibility, - emojis=config.EMOJIS.split(" "), - custom_emojis=EMOJIS_BY_NAME, + return htmlify( + render_template( + "new.html", + reply=reply_id, + content=content, + thread=thread, + visibility=ap.Visibility, + emojis=config.EMOJIS.split(" "), + custom_emojis=EMOJIS_BY_NAME, + ) ) @@ -262,7 +275,7 @@ def admin_new() -> _Response: def admin_lists() -> _Response: lists = list(DB.lists.find()) - return render_template("lists.html", lists=lists) + return htmlify(render_template("lists.html", lists=lists)) @blueprint.route("/admin/notifications") @@ -341,12 +354,14 @@ def admin_notifications() -> _Response: inbox_data, reverse=True, key=lambda doc: doc["_id"].generation_time ) - return render_template( - "stream.html", - inbox_data=inbox_data, - older_than=older_than, - newer_than=newer_than, - nid=nid, + return htmlify( + render_template( + "stream.html", + inbox_data=inbox_data, + older_than=older_than, + newer_than=newer_than, + nid=nid, + ) ) @@ -365,8 +380,10 @@ def admin_stream() -> _Response: DB.activities, q, limit=int(request.args.get("limit", 25)) ) - return render_template( - tpl, inbox_data=inbox_data, older_than=older_than, newer_than=newer_than + return htmlify( + render_template( + tpl, inbox_data=inbox_data, older_than=older_than, newer_than=newer_than + ) ) @@ -393,8 +410,10 @@ def admin_list(name: str) -> _Response: DB.activities, q, limit=int(request.args.get("limit", 25)) ) - return render_template( - tpl, inbox_data=inbox_data, older_than=older_than, newer_than=newer_than + return htmlify( + render_template( + tpl, inbox_data=inbox_data, older_than=older_than, newer_than=newer_than + ) ) @@ -413,8 +432,10 @@ def admin_bookmarks() -> _Response: DB.activities, q, limit=int(request.args.get("limit", 25)) ) - return render_template( - tpl, inbox_data=inbox_data, older_than=older_than, newer_than=newer_than + return htmlify( + render_template( + tpl, inbox_data=inbox_data, older_than=older_than, newer_than=newer_than + ) ) @@ -425,7 +446,7 @@ def u2f_register(): if request.method == "GET": payload = u2f.begin_registration(ID) session["challenge"] = payload - return render_template("u2f.html", payload=payload) + return htmlify(render_template("u2f.html", payload=payload)) else: resp = json.loads(request.form.get("resp")) device, device_cert = u2f.complete_registration(session["challenge"], resp) @@ -439,8 +460,10 @@ def u2f_register(): @login_required def authorize_follow(): if request.method == "GET": - return render_template( - "authorize_remote_follow.html", profile=request.args.get("profile") + return htmlify( + render_template( + "authorize_remote_follow.html", profile=request.args.get("profile") + ) ) actor = get_actor_url(request.form.get("profile")) diff --git a/blueprints/indieauth.py b/blueprints/indieauth.py index 9514ec3..08188e5 100644 --- a/blueprints/indieauth.py +++ b/blueprints/indieauth.py @@ -19,6 +19,7 @@ from itsdangerous import BadSignature from config import DB from config import JWT from core.shared import _get_ip +from core.shared import htmlify from core.shared import login_required blueprint = flask.Blueprint("indieauth", __name__) @@ -105,15 +106,17 @@ def indieauth_endpoint(): scope = request.args.get("scope", "").split() print("STATE", state) - return render_template( - "indieauth_flow.html", - client=get_client_id_data(client_id), - scopes=scope, - redirect_uri=redirect_uri, - state=state, - response_type=response_type, - client_id=client_id, - me=me, + return htmlify( + render_template( + "indieauth_flow.html", + client=get_client_id_data(client_id), + scopes=scope, + redirect_uri=redirect_uri, + state=state, + response_type=response_type, + client_id=client_id, + me=me, + ) ) # Auth verification via POST diff --git a/blueprints/tasks.py b/blueprints/tasks.py index d2545dc..950e4c8 100644 --- a/blueprints/tasks.py +++ b/blueprints/tasks.py @@ -554,10 +554,14 @@ def task_process_reply() -> _Response: root_reply = in_reply_to + # Fetch the activity reply reply = ap.fetch_remote_activity(in_reply_to) if reply.has_type(ap.ActivityType.CREATE): reply = reply.get_object() + # Store some metadata for the UI + # FIXME(tsileo): be able to display: "In reply to @user@domain.tld"? + new_replies = [activity, reply] while 1: diff --git a/config.py b/config.py index 71523eb..3f8e0f7 100644 --- a/config.py +++ b/config.py @@ -8,7 +8,7 @@ from pathlib import Path import yaml from itsdangerous import JSONWebSignatureSerializer from little_boxes import strtobool -from little_boxes.activitypub import DEFAULT_CTX as AP_DEFAULT_CTX +from little_boxes.activitypub import CTX_AS as AP_DEFAULT_CTX from pymongo import MongoClient import sass diff --git a/core/activitypub.py b/core/activitypub.py index 2559f71..9cebffe 100644 --- a/core/activitypub.py +++ b/core/activitypub.py @@ -263,6 +263,9 @@ def post_to_outbox(activity: ap.BaseActivity) -> str: class MicroblogPubBackend(Backend): """Implements a Little Boxes backend, backed by MongoDB.""" + def ap_context(self) -> Any: + return DEFAULT_CTX + def base_url(self) -> str: return BASE_URL diff --git a/core/shared.py b/core/shared.py index 5ea6db9..77702ad 100644 --- a/core/shared.py +++ b/core/shared.py @@ -59,7 +59,19 @@ def build_resp(resp): return resp, headers -def jsonify(**data): +def htmlify(data): + resp, headers = build_resp(data) + return Response( + response=resp, + headers={ + **headers, + "Content-Type": "text/html; charset=utf-8", + "Cache-Control": "max-age=0, private, must-revalidate", + }, + ) + + +def activitypubify(**data): if "@context" not in data: data["@context"] = config.DEFAULT_CTX resp, headers = build_resp(json.dumps(data)) @@ -68,9 +80,7 @@ def jsonify(**data): headers={ **headers, "Cache-Control": "max-age=0, private, must-revalidate", - "Content-Type": "application/json" - if app.debug - else "application/activity+json", + "Content-Type": "application/activity+json", }, ) From 681cfe5e545c41fe8f81c729f3b57660a5cdbb49 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 24 Aug 2019 10:58:35 +0200 Subject: [PATCH 0648/1425] Cleanup JSON outputs --- app.py | 8 ++--- blueprints/api.py | 28 ++++++++--------- blueprints/indieauth.py | 8 ++--- blueprints/well_known.py | 68 ++++++++++++++++++---------------------- core/shared.py | 21 +++++-------- 5 files changed, 58 insertions(+), 75 deletions(-) diff --git a/app.py b/app.py index b0cb623..a3938a7 100644 --- a/app.py +++ b/app.py @@ -14,7 +14,6 @@ from flask import Flask from flask import Response from flask import abort from flask import g -from flask import jsonify as flask_jsonify from flask import redirect from flask import render_template from flask import request @@ -67,6 +66,7 @@ from core.shared import activitypubify from core.shared import csrf from core.shared import htmlify from core.shared import is_api_request +from core.shared import jsonify from core.shared import login_required from core.shared import noindex from core.shared import paginated_query @@ -173,7 +173,7 @@ def handle_value_error(error): logger.error( f"caught value error for {g.request_id}: {error!r}, {traceback.format_tb(error.__traceback__)}" ) - response = flask_jsonify(message=error.args[0], request_id=g.request_id) + response = jsonify({"message": error.args[0], "request_id": g.request_id}) response.status_code = 400 return response @@ -183,7 +183,7 @@ def handle_activitypub_error(error): logger.error( f"caught activitypub error for {g.request_id}: {error!r}, {traceback.format_tb(error.__traceback__)}" ) - response = flask_jsonify({**error.to_dict(), "request_id": g.request_id}) + response = jsonify({**error.to_dict(), "request_id": g.request_id}) response.status_code = error.status_code return response @@ -193,7 +193,7 @@ def handle_task_error(error): logger.error( f"caught activitypub error for {g.request_id}: {error!r}, {traceback.format_tb(error.__traceback__)}" ) - response = flask_jsonify({"traceback": error.message, "request_id": g.request_id}) + response = jsonify({"traceback": error.message, "request_id": g.request_id}) response.status_code = 500 return response diff --git a/blueprints/api.py b/blueprints/api.py index 51a5f39..e34a9bc 100644 --- a/blueprints/api.py +++ b/blueprints/api.py @@ -1,4 +1,3 @@ -import json import mimetypes from datetime import datetime from datetime import timedelta @@ -10,7 +9,6 @@ from typing import List import flask from bson.objectid import ObjectId -from flask import Response from flask import abort from flask import current_app as app from flask import redirect @@ -41,6 +39,7 @@ from core.meta import _meta from core.shared import MY_PERSON from core.shared import _Response from core.shared import csrf +from core.shared import jsonify from core.shared import login_required from core.tasks import Tasks from utils import emojis @@ -124,7 +123,7 @@ def _user_api_response(**kwargs) -> _Response: if _redirect: return redirect(_redirect) - resp = flask.jsonify(**kwargs) + resp = jsonify(kwargs) resp.status_code = 201 return resp @@ -132,7 +131,7 @@ def _user_api_response(**kwargs) -> _Response: @blueprint.route("/api/key") @login_required def api_user_key() -> _Response: - return flask.jsonify(api_key=ADMIN_API_KEY) + return jsonify({"api_key": ADMIN_API_KEY}) @blueprint.route("/note/delete", methods=["POST"]) @@ -575,25 +574,24 @@ def api_follow() -> _Response: def api_debug() -> _Response: """Endpoint used/needed for testing, only works in DEBUG_MODE.""" if not DEBUG_MODE: - return flask.jsonify(message="DEBUG_MODE is off") + return jsonify({"message": "DEBUG_MODE is off"}) if request.method == "DELETE": _drop_db() - return flask.jsonify(message="DB dropped") + return jsonify(dict(message="DB dropped")) - return flask.jsonify( - inbox=DB.activities.count({"box": Box.INBOX.value}), - outbox=DB.activities.count({"box": Box.OUTBOX.value}), - outbox_data=without_id(DB.activities.find({"box": Box.OUTBOX.value})), + return jsonify( + dict( + inbox=DB.activities.count({"box": Box.INBOX.value}), + outbox=DB.activities.count({"box": Box.OUTBOX.value}), + outbox_data=without_id(DB.activities.find({"box": Box.OUTBOX.value})), + ) ) @blueprint.route("/stream") @api_required def api_stream() -> _Response: - return Response( - response=json.dumps( - feed.build_inbox_json_feed("/api/stream", request.args.get("cursor")) - ), - headers={"Content-Type": "application/json"}, + return jsonify( + feed.build_inbox_json_feed("/api/stream", request.args.get("cursor")) ) diff --git a/blueprints/indieauth.py b/blueprints/indieauth.py index 08188e5..76ee66c 100644 --- a/blueprints/indieauth.py +++ b/blueprints/indieauth.py @@ -1,5 +1,4 @@ import binascii -import json import os from datetime import datetime from datetime import timedelta @@ -20,6 +19,7 @@ from config import DB from config import JWT from core.shared import _get_ip from core.shared import htmlify +from core.shared import jsonify from core.shared import login_required blueprint = flask.Blueprint("indieauth", __name__) @@ -27,11 +27,7 @@ blueprint = flask.Blueprint("indieauth", __name__) def build_auth_resp(payload): if request.headers.get("Accept") == "application/json": - return Response( - status=200, - headers={"Content-Type": "application/json"}, - response=json.dumps(payload), - ) + return jsonify(payload) return Response( status=200, headers={"Content-Type": "application/x-www-form-urlencoded"}, diff --git a/blueprints/well_known.py b/blueprints/well_known.py index 1f936c0..bf152f2 100644 --- a/blueprints/well_known.py +++ b/blueprints/well_known.py @@ -1,9 +1,7 @@ -import json import mimetypes from typing import Any import flask -from flask import Response from flask import abort from flask import request from little_boxes import activitypub as ap @@ -11,6 +9,7 @@ from little_boxes import activitypub as ap import config from config import DB from core.meta import Box +from core.shared import jsonify blueprint = flask.Blueprint("well_known", __name__) @@ -45,22 +44,21 @@ def wellknown_webfinger() -> Any: ], } - return Response( - response=json.dumps(out), - headers={"Content-Type": "application/jrd+json; charset=utf-8"}, - ) + return jsonify(out, "application/jrd+json; charset=utf-8") @blueprint.route("/.well-known/nodeinfo") def wellknown_nodeinfo() -> Any: """Exposes the NodeInfo endpoint (http://nodeinfo.diaspora.software/).""" - return flask.jsonify( - links=[ - { - "rel": "http://nodeinfo.diaspora.software/ns/schema/2.1", - "href": f"{config.ID}/nodeinfo", - } - ] + return jsonify( + { + "links": [ + { + "rel": "http://nodeinfo.diaspora.software/ns/schema/2.1", + "href": f"{config.ID}/nodeinfo", + } + ] + } ) @@ -73,29 +71,25 @@ def nodeinfo() -> Any: "type": {"$in": [ap.ActivityType.CREATE.value, ap.ActivityType.ANNOUNCE.value]}, } - response = json.dumps( - { - "version": "2.1", - "software": { - "name": "microblogpub", - "version": config.VERSION, - "repository": "https://github.com/tsileo/microblog.pub", - }, - "protocols": ["activitypub"], - "services": {"inbound": [], "outbound": []}, - "openRegistrations": False, - "usage": {"users": {"total": 1}, "localPosts": DB.activities.count(q)}, - "metadata": { - "nodeName": f"@{config.USERNAME}@{config.DOMAIN}", - "version": config.VERSION, - "versionDate": config.VERSION_DATE, - }, - } - ) - - return Response( - headers={ - "Content-Type": "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.1#" + out = { + "version": "2.1", + "software": { + "name": "microblogpub", + "version": config.VERSION, + "repository": "https://github.com/tsileo/microblog.pub", }, - response=response, + "protocols": ["activitypub"], + "services": {"inbound": [], "outbound": []}, + "openRegistrations": False, + "usage": {"users": {"total": 1}, "localPosts": DB.activities.count(q)}, + "metadata": { + "nodeName": f"@{config.USERNAME}@{config.DOMAIN}", + "version": config.VERSION, + "versionDate": config.VERSION_DATE, + }, + } + + return jsonify( + out, + "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.1#", ) diff --git a/core/shared.py b/core/shared.py index 77702ad..f21676f 100644 --- a/core/shared.py +++ b/core/shared.py @@ -48,7 +48,7 @@ MY_PERSON = ap.Person(**ME) @lru_cache(512) def build_resp(resp): """Encode the response to gzip if supported by the client.""" - headers = {} + headers = {"Cache-Control": "max-age=0, private, must-revalidate"} accept_encoding = request.headers.get("Accept-Encoding", "") if "gzip" in accept_encoding.lower(): return ( @@ -59,15 +59,15 @@ def build_resp(resp): return resp, headers +def jsonify(data, content_type="application/json"): + resp, headers = build_resp(json.dumps(data)) + return Response(headers={**headers, "Content-Type": content_type}, response=resp) + + def htmlify(data): resp, headers = build_resp(data) return Response( - response=resp, - headers={ - **headers, - "Content-Type": "text/html; charset=utf-8", - "Cache-Control": "max-age=0, private, must-revalidate", - }, + response=resp, headers={**headers, "Content-Type": "text/html; charset=utf-8"} ) @@ -76,12 +76,7 @@ def activitypubify(**data): data["@context"] = config.DEFAULT_CTX resp, headers = build_resp(json.dumps(data)) return Response( - response=resp, - headers={ - **headers, - "Cache-Control": "max-age=0, private, must-revalidate", - "Content-Type": "application/activity+json", - }, + response=resp, headers={**headers, "Content-Type": "application/activity+json"} ) From 40c847f1bd2e3a0aeab42d7eca4413bd7505becf Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 24 Aug 2019 11:08:19 +0200 Subject: [PATCH 0649/1425] Fix emoji loading --- utils/emojis.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/utils/emojis.py b/utils/emojis.py index f0c9539..5d04071 100644 --- a/utils/emojis.py +++ b/utils/emojis.py @@ -11,8 +11,8 @@ from little_boxes import activitypub as ap EMOJI_REGEX = re.compile(r"(:[\d\w]+:)") -EMOJIS: Dict[str, ap.Emoji] = {} -EMOJIS_BY_NAME: Dict[str, ap.Emoji] = {} +EMOJIS: Dict[str, Dict[str, Any]] = {} +EMOJIS_BY_NAME: Dict[str, Dict[str, Any]] = {} def _load_emojis(root_dir: Path, base_url: str) -> None: @@ -22,7 +22,8 @@ def _load_emojis(root_dir: Path, base_url: str) -> None: mt = mimetypes.guess_type(emoji.name)[0] if mt and mt.startswith("image/"): name = emoji.name.split(".")[0] - ap_emoji = ap.Emoji( + ap_emoji = dict( + type=ap.ActivityType.EMOJI.value, name=f":{name}:", updated=ap.format_datetime(datetime.fromtimestamp(0.0).astimezone()), id=f"{base_url}/emoji/{name}", @@ -33,7 +34,7 @@ def _load_emojis(root_dir: Path, base_url: str) -> None: }, ) EMOJIS[emoji.name] = ap_emoji - EMOJIS_BY_NAME[ap_emoji.name] = ap_emoji + EMOJIS_BY_NAME[ap_emoji["name"]] = ap_emoji def tags(content: str) -> List[Dict[str, Any]]: @@ -41,7 +42,7 @@ def tags(content: str) -> List[Dict[str, Any]]: added: Set[str] = set() for e in re.findall(EMOJI_REGEX, content): if e not in added and e in EMOJIS_BY_NAME: - tags.append(EMOJIS_BY_NAME[e].to_dict()) + tags.append(EMOJIS_BY_NAME[e]) added.add(e) return tags From a8d571f2a330a28af39117be81ff0b6d7aeff194 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 24 Aug 2019 11:11:36 +0200 Subject: [PATCH 0650/1425] More bugfixes --- app.py | 4 +--- blueprints/admin.py | 4 +++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app.py b/app.py index a3938a7..bc16091 100644 --- a/app.py +++ b/app.py @@ -508,9 +508,7 @@ def outbox(): @app.route("/emoji/") def ap_emoji(name): if name in EMOJIS: - return activitypubify( - **{**EMOJIS[name].to_dict(), "@context": config.DEFAULT_CTX} - ) + return activitypubify(**{**EMOJIS[name], "@context": config.DEFAULT_CTX}) abort(404) diff --git a/blueprints/admin.py b/blueprints/admin.py index 3bae40e..79993df 100644 --- a/blueprints/admin.py +++ b/blueprints/admin.py @@ -265,7 +265,9 @@ def admin_new() -> _Response: thread=thread, visibility=ap.Visibility, emojis=config.EMOJIS.split(" "), - custom_emojis=EMOJIS_BY_NAME, + custom_emojis={ + name: ap.Emoji(**dat) for name, dat in EMOJIS_BY_NAME.items() + }, ) ) From 2dcc7319ea05ecef1b32efbee1349718cf2aeaff Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 24 Aug 2019 11:18:01 +0200 Subject: [PATCH 0651/1425] Add Cache-Control back --- core/shared.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/shared.py b/core/shared.py index f21676f..e1ab120 100644 --- a/core/shared.py +++ b/core/shared.py @@ -53,7 +53,7 @@ def build_resp(resp): if "gzip" in accept_encoding.lower(): return ( gzip.compress(resp.encode(), compresslevel=6), - {"Vary": "Accept-Encoding", "Content-Encoding": "gzip"}, + {**headers, "Vary": "Accept-Encoding", "Content-Encoding": "gzip"}, ) return resp, headers From 14e5e6b7e7e4453880f65fc256e4d68260b457e8 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 24 Aug 2019 12:09:45 +0200 Subject: [PATCH 0652/1425] Fix collections --- core/activitypub.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/activitypub.py b/core/activitypub.py index 9cebffe..27af107 100644 --- a/core/activitypub.py +++ b/core/activitypub.py @@ -470,7 +470,7 @@ def build_ordered_collection( # Returns an empty page if there's a cursor if cursor: return { - "@context": ap.COLLECTION_CTX, + "@context": DEFAULT_CTX, "type": ap.ActivityType.ORDERED_COLLECTION_PAGE.value, "id": BASE_URL + "/" + col_name + "?cursor=" + cursor, "partOf": BASE_URL + "/" + col_name, @@ -478,7 +478,7 @@ def build_ordered_collection( "orderedItems": [], } return { - "@context": ap.COLLECTION_CTX, + "@context": DEFAULT_CTX, "id": BASE_URL + "/" + col_name, "totalItems": 0, "type": ap.ActivityType.ORDERED_COLLECTION.value, @@ -496,7 +496,7 @@ def build_ordered_collection( # No cursor, this is the first page and we return an OrderedCollection if not cursor: resp = { - "@context": ap.COLLECTION_CTX, + "@context": DEFAULT_CTX, "id": f"{BASE_URL}/{col_name}", "totalItems": total_items, "type": ap.ActivityType.ORDERED_COLLECTION.value, @@ -521,7 +521,7 @@ def build_ordered_collection( # If there's a cursor, then we return an OrderedCollectionPage resp = { - "@context": ap.COLLECTION_CTX, + "@context": DEFAULT_CTX, "type": ap.ActivityType.ORDERED_COLLECTION_PAGE.value, "id": BASE_URL + "/" + col_name + "?cursor=" + start_cursor, "totalItems": total_items, From 562aae86f3e290c319ae9c792e9fecd821666f33 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 25 Aug 2019 00:16:39 +0200 Subject: [PATCH 0653/1425] Code highlighting --- app.py | 2 ++ requirements.txt | 1 + templates/layout.html | 1 + templates/utils.html | 4 ++-- utils/highlight.py | 30 ++++++++++++++++++++++++++++++ utils/template_filters.py | 6 ++++++ 6 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 utils/highlight.py diff --git a/app.py b/app.py index bc16091..bf2f838 100644 --- a/app.py +++ b/app.py @@ -72,6 +72,7 @@ from core.shared import noindex from core.shared import paginated_query from utils.blacklist import is_blacklisted from utils.emojis import EMOJIS +from utils.highlight import HIGHLIGHT_CSS from utils.key import get_secret_key from utils.template_filters import filters @@ -153,6 +154,7 @@ def inject_config(): else 0, me=ME, base_url=config.BASE_URL, + highlight_css=HIGHLIGHT_CSS, ) diff --git a/requirements.txt b/requirements.txt index 3e81189..5c0a865 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,3 +24,4 @@ pyyaml pillow emoji-unicode html5lib +Pygments diff --git a/templates/layout.html b/templates/layout.html index a71eab7..be29498 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -21,6 +21,7 @@ width: 25px; height: 25px; } +{{ highlight_css }} {% block headers %}{% endblock %} diff --git a/templates/utils.html b/templates/utils.html index caeace5..10a2303 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -77,7 +77,7 @@ {% if obj | has_type(['Article', 'Page']) %} {{ obj.name }} {{ obj | url_or_id | get_url }} {% elif obj | has_type('Question') %} - {{ obj.content | clean | replace_custom_emojis(obj) | safe }} + {{ obj.content | clean | replace_custom_emojis(obj) | code_highlight | safe }}
            @@ -146,7 +146,7 @@ {% else %} - {{ obj.content | clean | replace_custom_emojis(obj) | safe }} + {{ obj.content | clean | replace_custom_emojis(obj) | code_highlight | safe }} {% endif %}
      diff --git a/utils/highlight.py b/utils/highlight.py new file mode 100644 index 0000000..74fa16f --- /dev/null +++ b/utils/highlight.py @@ -0,0 +1,30 @@ +from functools import lru_cache + +from bs4 import BeautifulSoup +from pygments import highlight as phighlight +from pygments.formatters import HtmlFormatter +from pygments.lexers import guess_lexer + +from config import THEME_STYLE +from config import ThemeStyle + +_FORMATTER = HtmlFormatter( + style="default" if THEME_STYLE == ThemeStyle.LIGHT else "vim" +) + +HIGHLIGHT_CSS = _FORMATTER.get_style_defs() + + +@lru_cache(512) +def highlight(html: str) -> str: + soup = BeautifulSoup(html, "html5lib") + for code in soup.find_all("code"): + if not code.parent.name == "pre": + continue + lexer = guess_lexer(code.text) + tag = BeautifulSoup(phighlight(code.text, lexer, _FORMATTER)).body.next + pre = code.parent + pre.replaceWith(tag) + out = soup.body + out.name = "div" + return str(out) diff --git a/utils/template_filters.py b/utils/template_filters.py index 11987d1..0e42231 100644 --- a/utils/template_filters.py +++ b/utils/template_filters.py @@ -21,6 +21,7 @@ from config import ID from config import MEDIA_CACHE from core.activitypub import _answer_key from utils import parse_datetime +from utils.highlight import highlight from utils.media import Kind from utils.media import _is_img @@ -47,6 +48,11 @@ def visibility_is_public(v: str) -> bool: return v in [ap.Visibility.PUBLIC.name, ap.Visibility.UNLISTED.name] +@filters.app_template_filter() +def code_highlight(content): + return highlight(content) + + @filters.app_template_filter() def emojify(text): return emoji_unicode.replace( From 4216b0b85739f7489525e2701e95bc032e6bfaed Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 25 Aug 2019 10:54:17 +0200 Subject: [PATCH 0654/1425] No more extra inboxes --- config.py | 1 - core/activitypub.py | 4 ---- 2 files changed, 5 deletions(-) diff --git a/config.py b/config.py index 3f8e0f7..7a44fac 100644 --- a/config.py +++ b/config.py @@ -67,7 +67,6 @@ with open(os.path.join(KEY_DIR, "me.yml")) as f: SUMMARY = conf["summary"] ICON_URL = conf["icon_url"] PASS = conf["pass"] - EXTRA_INBOXES = conf.get("extra_inboxes", []) HIDE_FOLLOWING = conf.get("hide_following", True) diff --git a/core/activitypub.py b/core/activitypub.py index 27af107..beeeffe 100644 --- a/core/activitypub.py +++ b/core/activitypub.py @@ -25,7 +25,6 @@ from little_boxes.httpsig import HTTPSigAuth from config import BASE_URL from config import DB from config import DEFAULT_CTX -from config import EXTRA_INBOXES from config import ID from config import KEY from config import ME @@ -276,9 +275,6 @@ class MicroblogPubBackend(Backend): """Setup a custom user agent.""" return USER_AGENT - def extra_inboxes(self) -> List[str]: - return EXTRA_INBOXES - def followers(self) -> List[str]: q = { "box": Box.INBOX.value, From bb43aec4741b5ab977076eccc3ba8b2cd4d90bb6 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 25 Aug 2019 11:09:34 +0200 Subject: [PATCH 0655/1425] Fix for local poll answers --- core/activitypub.py | 8 +++++++- core/meta.py | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/core/activitypub.py b/core/activitypub.py index beeeffe..2f6f693 100644 --- a/core/activitypub.py +++ b/core/activitypub.py @@ -31,6 +31,7 @@ from config import ME from config import USER_AGENT from core.db import find_one_activity from core.db import update_many_activities +from core.db import update_one_activity from core.meta import Box from core.meta import MetaKey from core.meta import by_object_id @@ -682,7 +683,7 @@ def handle_question_reply(create: ap.Create, question: ap.Question) -> None: by_remote_id(create.id), { "$set": { - "meta.answer_to": question.id, + "meta.poll_answer_to": question.id, "meta.poll_answer_choice": choice, "meta.stream": False, "meta.poll_answer": True, @@ -727,6 +728,11 @@ def handle_replies(create: ap.Create) -> None: } }, ) + # Mark our reply as a poll answers, to "hide" it from the UI + update_one_activity( + by_remote_id(create.id), + upsert({MetaKey.POLL_ANSWER: True, MetaKey.POLL_ANSWER_TO: reply.id}), + ) return None # It's a regular reply, try to increment the reply counter diff --git a/core/meta.py b/core/meta.py index 6a6ccef..422437c 100644 --- a/core/meta.py +++ b/core/meta.py @@ -24,6 +24,7 @@ class MetaKey(Enum): NOTIFICATION_UNREAD = "notification_unread" NOTIFICATION_FOLLOWS_BACK = "notification_follows_back" POLL_ANSWER = "poll_answer" + POLL_ANSWER_TO = "poll_answer_to" STREAM = "stream" ACTOR_ID = "actor_id" ACTOR = "actor" From 3f65368547ad4c3e4dfb9b81e75653d08238c3bc Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 25 Aug 2019 12:48:04 +0200 Subject: [PATCH 0656/1425] Improve replies handling --- blueprints/tasks.py | 26 ++++++++++++++++---------- core/activitypub.py | 11 ++++++++++- core/meta.py | 3 +++ 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/blueprints/tasks.py b/blueprints/tasks.py index 950e4c8..d6c463b 100644 --- a/blueprints/tasks.py +++ b/blueprints/tasks.py @@ -2,6 +2,8 @@ import json import traceback from datetime import datetime from datetime import timezone +from typing import Any +from typing import Dict import flask import requests @@ -559,9 +561,6 @@ def task_process_reply() -> _Response: if reply.has_type(ap.ActivityType.CREATE): reply = reply.get_object() - # Store some metadata for the UI - # FIXME(tsileo): be able to display: "In reply to @user@domain.tld"? - new_replies = [activity, reply] while 1: @@ -579,29 +578,36 @@ def task_process_reply() -> _Response: app.logger.info(f"root_reply={reply!r} for activity={activity!r}") - # Ensure the "root reply" is present in the inbox/outbox - if not find_one_activity( - {**by_object_id(root_reply), **by_type(ap.ActivityType.CREATE)} - ): - return "" - # In case the activity was from the inbox update_one_activity( {**by_object_id(activity.id), **by_type(ap.ActivityType.CREATE)}, upsert({MetaKey.THREAD_ROOT_PARENT: root_reply}), ) - for new_reply in new_replies: + for (new_reply_idx, new_reply) in enumerate(new_replies): if find_one_activity( {**by_object_id(new_reply.id), **by_type(ap.ActivityType.CREATE)} ) or DB.replies.find_one(by_remote_id(new_reply.id)): continue actor = new_reply.get_actor() + is_root_reply = new_reply_idx == len(new_replies) - 1 + if is_root_reply: + reply_flags: Dict[str, Any] = {} + else: + reply_actor = new_replies[new_reply_idx + 1].get_actor() + is_in_reply_to_self = actor.id == reply_actor.id + reply_flags = {MetaKey.IN_REPLY_TO_SELF.value: is_in_reply_to_self} + if not is_in_reply_to_self: + reply_flags[MetaKey.IN_REPLY_TO_ACTOR.value] = reply_actor.to_dict( + embed=True + ) + # Save the reply with the cached actor and the thread flag/ID save_reply( new_reply, { + **reply_flags, MetaKey.THREAD_ROOT_PARENT.value: root_reply, MetaKey.ACTOR.value: actor.to_dict(embed=True), MetaKey.ACTOR_HASH.value: _actor_hash(actor), diff --git a/core/activitypub.py b/core/activitypub.py index 2f6f693..2f064f2 100644 --- a/core/activitypub.py +++ b/core/activitypub.py @@ -735,6 +735,15 @@ def handle_replies(create: ap.Create) -> None: ) return None + # Update the activity to save some data about the reply + if reply.get_actor().id == create.get_actor().id: + in_reply_to_data = {MetaKey.IN_REPLY_TO_SELF: True} + else: + in_reply_to_data = { + MetaKey.IN_REPLY_TO_ACTOR: reply.get_actor().to_dict(embed=True) + } + update_one_activity(by_remote_id(create.id), upsert(in_reply_to_data)) + # It's a regular reply, try to increment the reply counter creply = DB.activities.find_one_and_update( {**by_object_id(in_reply_to), **by_type(ap.ActivityType.CREATE)}, @@ -747,4 +756,4 @@ def handle_replies(create: ap.Create) -> None: ) # Spawn a task to process it (and determine if it needs to be saved) - Tasks.process_reply(create.get_object().id) + Tasks.process_reply(create.get_object_id()) diff --git a/core/meta.py b/core/meta.py index 422437c..f5dde02 100644 --- a/core/meta.py +++ b/core/meta.py @@ -40,6 +40,9 @@ class MetaKey(Enum): PUBLIC = "public" THREAD_ROOT_PARENT = "thread_root_parent" + IN_REPLY_TO_SELF = "in_reply_to_self" + IN_REPLY_TO_ACTOR = "in_reply_to_actor" + SERVER = "server" VISIBILITY = "visibility" OBJECT_VISIBILITY = "object_visibility" From 3bd2ab8102392b82b20f3feadeb5b05d4c3e1747 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 25 Aug 2019 16:02:05 +0200 Subject: [PATCH 0657/1425] Add new internal profile page for remote actors --- blueprints/admin.py | 43 ++++++++++++++++++ templates/stream.html | 95 +++++++++++++++++++++++++++++++++++++++ templates/utils.html | 4 +- utils/template_filters.py | 2 - 4 files changed, 140 insertions(+), 4 deletions(-) diff --git a/blueprints/admin.py b/blueprints/admin.py index 79993df..ed65c15 100644 --- a/blueprints/admin.py +++ b/blueprints/admin.py @@ -207,6 +207,49 @@ def admin_lookup() -> _Response: ) +@blueprint.route("/admin/profile", methods=["GET"]) +@login_required +def admin_profile() -> _Response: + if not request.args.get("actor_id"): + abort(404) + + actor_id = request.args.get("actor_id") + actor = ap.fetch_remote_activity(actor_id) + q = { + "meta.actor_id": actor_id, + "box": "inbox", + "type": {"$in": [ap.ActivityType.CREATE.value, ap.ActivityType.ANNOUNCE.value]}, + } + inbox_data, older_than, newer_than = paginated_query( + DB.activities, q, limit=int(request.args.get("limit", 25)) + ) + follower = find_one_activity( + { + "box": "inbox", + "type": ap.ActivityType.FOLLOW.value, + "meta.actor_id": actor.id, + "meta.undo": False, + } + ) + following = find_one_activity( + {"type": ap.ActivityType.ACCEPT.value, "meta.actor_id": actor.id} + ) + + return htmlify( + render_template( + "stream.html", + actor_id=actor_id, + actor=actor.to_dict(), + inbox_data=inbox_data, + older_than=older_than, + newer_than=newer_than, + follower=follower, + following=following, + lists=list(DB.lists.find()), + ) + ) + + @blueprint.route("/admin/thread") @login_required def admin_thread() -> _Response: diff --git a/templates/stream.html b/templates/stream.html index eccb207..2e71f45 100644 --- a/templates/stream.html +++ b/templates/stream.html @@ -17,6 +17,101 @@
    {% endif %} +{% if actor %} +{% set actor_redir = request.path + "?actor_id=" + request.args.get('actor_id') %} + +
    +
    + {% if follower %}follows you!{% endif %} + +{% if following %} +
    + + + + +
    +
    + + + + +
    + + +{% if lists %} +
    + + + + + +
    +{% endif %} + +{% for l in lists %} +{% if actor.id in l.members %} +
    + + + + +
    + + +{% endif %} +{% endfor %} + + + + +{% else %} +
    + + + + +
    +
    + + + + +
    + + +{% endif %} +
    + + + +{% if not actor.icon %} + +{% else %} +{% endif %} + +
    +
    {{ (actor.name or actor.preferredUsername) | clean | replace_custom_emojis(actor) | safe }}
    +@{{ actor.preferredUsername }}@{{ actor | url_or_id | get_url | domain }} +
    +     + +{% if actor.summary %} +
    + {{ actor.summary | clean | replace_custom_emojis(actor) | safe }} +
    +{% endif %} +
    + +{% endif %} +
    {% for item in inbox_data %} {% if 'actor' in item.meta %} diff --git a/templates/utils.html b/templates/utils.html index 10a2303..3b8285d 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -322,8 +322,8 @@ -{% endif %} +profile +{% endif %} {% endif %} diff --git a/utils/template_filters.py b/utils/template_filters.py index 0e42231..4467a24 100644 --- a/utils/template_filters.py +++ b/utils/template_filters.py @@ -99,8 +99,6 @@ ALLOWED_TAGS = [ @filters.app_template_filter() def replace_custom_emojis(content, note): - print("\n" * 50) - print("custom_replace", note) idx = {} for tag in note.get("tag", []): if tag.get("type") == "Emoji": From 376de7b23a1bc42b1a270255342a15ffb939ad96 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 25 Aug 2019 16:08:12 +0200 Subject: [PATCH 0658/1425] Display key values in profiles --- templates/stream.html | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/templates/stream.html b/templates/stream.html index 2e71f45..a9736b8 100644 --- a/templates/stream.html +++ b/templates/stream.html @@ -1,6 +1,6 @@ {% extends "layout.html" %} {% import 'utils.html' as utils %} -{% block title %}{% if request.path == url_for('admin.admin_stream') %}Stream{% else %}Notifications{% endif %} - {{ config.NAME }}{% endblock %} +{% block title %}{% if request.path == url_for('admin.admin_stream') %}Stream{% elif actor_id %}Profile {{ actor.name }}{% else %}Notifications{% endif %} - {{ config.NAME }}{% endblock %} {% block content %}
    {% include "header.html" %} @@ -108,6 +108,16 @@ {{ actor.summary | clean | replace_custom_emojis(actor) | safe }}
    {% endif %} + +{% if actor.attachment %} +
      + {% for item in actor.attachment %} + {% if item.type == "PropertyValue" %} +
    • {{ item.name }}: {{ item.value | clean | replace_custom_emojis(actor) | safe }}
      • + {% endif %} + {% endfor %} +
    +{% endif %}
    {% endif %} From 0dd5588faaf301035bfe36bc34ddc480ea6551a0 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 25 Aug 2019 21:29:07 +0200 Subject: [PATCH 0659/1425] Tweak the UI --- templates/utils.html | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/templates/utils.html b/templates/utils.html index 3b8285d..8514866 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -60,7 +60,7 @@ {% if not perma %} - + {% endif %}
    @@ -316,18 +316,16 @@ {% endif %} {% else %} -
    - - - - -
    profile {% endif %} {% endif %} +{% if obj | url_or_id | is_from_outbox %} permalink +{% else %} +source +{% endif %} {% if session.logged_in %} {{ meta.object_visibility | visibility }} {% endif %} From ab623b0f3a841f733795abd6f4664ab96a444820 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 27 Aug 2019 00:16:18 +0200 Subject: [PATCH 0660/1425] Tweak the HTML whitelist --- utils/template_filters.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/template_filters.py b/utils/template_filters.py index 4467a24..085791c 100644 --- a/utils/template_filters.py +++ b/utils/template_filters.py @@ -75,6 +75,9 @@ ALLOWED_TAGS = [ "li", "ol", "strong", + "sup", + "sub", + "del", "ul", "span", "div", From bb62ebc2bcf956884423095a9d9cae32365eb39b Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 27 Aug 2019 00:24:16 +0200 Subject: [PATCH 0661/1425] Hide source button when an activity is not public --- templates/utils.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/utils.html b/templates/utils.html index 8514866..87c8109 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -321,6 +321,7 @@ {% endif %} +{% if meta.object_visibility | visibility_is_public %} {% if obj | url_or_id | is_from_outbox %} permalink {% else %} @@ -329,6 +330,7 @@ {% if session.logged_in %} {{ meta.object_visibility | visibility }} {% endif %} +{% endif %}
    From 6161da4210bce13a1c6def65f35e6816ddaa7745 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 27 Aug 2019 00:32:22 +0200 Subject: [PATCH 0662/1425] Fix UI --- templates/utils.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/utils.html b/templates/utils.html index 87c8109..1ead28a 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -327,10 +327,10 @@ {% else %} source {% endif %} +{% endif %} {% if session.logged_in %} {{ meta.object_visibility | visibility }} {% endif %} -{% endif %}
    From 885af6ae6e8b8fc2cc9d15c848ef2cb9aafc7fbd Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 27 Aug 2019 23:02:19 +0200 Subject: [PATCH 0663/1425] Add more profile links --- sass/base_theme.scss | 1 + templates/followers.html | 5 +++++ templates/following.html | 1 + templates/lookup.html | 2 ++ templates/utils.html | 1 - 5 files changed, 9 insertions(+), 1 deletion(-) diff --git a/sass/base_theme.scss b/sass/base_theme.scss index d8bf0f9..437095b 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -284,6 +284,7 @@ a:hover { .bar-item:hover { background: $primary-color; color: $background-color; + text-decoration: none; } .bar-item-no-border { color: $color-light; diff --git a/templates/followers.html b/templates/followers.html index b492e29..f1bba01 100644 --- a/templates/followers.html +++ b/templates/followers.html @@ -9,6 +9,11 @@
    {% for follower in followers_data %} + {% if session.logged_in %} + + {% endif %}
    {{ utils.display_actor_inline(follower, size=80) }}
    diff --git a/templates/following.html b/templates/following.html index fff0d8d..f084d32 100644 --- a/templates/following.html +++ b/templates/following.html @@ -11,6 +11,7 @@ {% for (follow_id, follow) in following_data %} {% if session.logged_in %}
    +profile
    diff --git a/templates/lookup.html b/templates/lookup.html index 07873c2..21a19a9 100644 --- a/templates/lookup.html +++ b/templates/lookup.html @@ -16,6 +16,7 @@
    {% if data | has_actor_type %}
    +profile @@ -26,6 +27,7 @@ {{ utils.display_actor_inline(data, size=80) }} + {% elif data | has_type('Create') %} {{ utils.display_note(data.object, ui=True) }} {% elif data | has_type(['Note', 'Article', 'Video', 'Audio', 'Page', 'Question']) %} diff --git a/templates/utils.html b/templates/utils.html index 1ead28a..b51426a 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -316,7 +316,6 @@ {% endif %} {% else %} -profile {% endif %} {% endif %} From 6e11ecce5b9de292ee0ce7183c9cf241f0b45986 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 28 Aug 2019 23:11:40 +0200 Subject: [PATCH 0664/1425] Delay poussetaches processing at startup --- blueprints/admin.py | 1 + docker-compose.yml | 4 +++- run.sh | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/blueprints/admin.py b/blueprints/admin.py index ed65c15..c304a46 100644 --- a/blueprints/admin.py +++ b/blueprints/admin.py @@ -51,6 +51,7 @@ def verify_pass(pwd): @blueprint.route("/admin/update_actor") @login_required def admin_update_actor() -> _Response: + # FIXME(tsileo): make this a task, and keep track of our own actor_hash at startup update = ap.Update( actor=MY_PERSON.id, object=MY_PERSON.to_dict(), diff --git a/docker-compose.yml b/docker-compose.yml index 40d0313..f04be1b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ services: web: image: 'microblogpub:latest' ports: - - "${WEB_PORT}:5005" + - "127.0.0.1:${WEB_PORT}:5005" volumes: - "${CONFIG_DIR}:/app/config" - "./static:/app/static" @@ -23,3 +23,5 @@ services: - "${DATA_DIR}/poussetaches:/app/poussetaches_data" environment: - POUSSETACHES_AUTH_KEY=${POUSSETACHES_AUTH_KEY} +# ports: +# - "127.0.0.1:${POUSSETACHES_PORT}:7991" diff --git a/run.sh b/run.sh index b3fa579..11f11fb 100755 --- a/run.sh +++ b/run.sh @@ -1,4 +1,5 @@ #!/bin/bash python -c "import logging; logging.basicConfig(level=logging.DEBUG); from core import migrations; migrations.perform()" python -c "from core import indexes; indexes.create_indexes()" +(sleep 5 && curl -X POST -u :$POUSETACHES_AUTH_KEY $MICROBLOGPUB_POUSSETACHES_HOST/resume)& gunicorn -t 600 -w 5 -b 0.0.0.0:5005 --log-level debug app:app From fa1cc4cb4c89ba4aa76660baa346af3009cfa225 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Wed, 28 Aug 2019 23:15:01 +0200 Subject: [PATCH 0665/1425] Re-add profile button under notes --- templates/utils.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/templates/utils.html b/templates/utils.html index b51426a..34b5eb0 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -325,6 +325,9 @@ permalink {% else %} source +{% if session.logged_in %} +profile +{% endif %} {% endif %} {% endif %} {% if session.logged_in %} From a49ba89a7e0a950a07982fb061bcb540dfb31e85 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Sep 2019 10:26:25 +0200 Subject: [PATCH 0666/1425] Improve follower/following management --- .dockerignore | 2 ++ core/activitypub.py | 17 ++++++++++ core/inbox.py | 29 ++++++++++++++++ core/meta.py | 18 ++++++++++ core/migrations.py | 78 +++++++++++++++++++++++++++++++++++++++++-- core/notifications.py | 32 +++++++++--------- 6 files changed, 157 insertions(+), 19 deletions(-) diff --git a/.dockerignore b/.dockerignore index ccc0367..f24c558 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,5 @@ +.git/ __pycache__/ data/ +data2/ tests/ diff --git a/core/activitypub.py b/core/activitypub.py index 2f064f2..65e488c 100644 --- a/core/activitypub.py +++ b/core/activitypub.py @@ -33,6 +33,7 @@ from core.db import find_one_activity from core.db import update_many_activities from core.db import update_one_activity from core.meta import Box +from core.meta import FollowStatus from core.meta import MetaKey from core.meta import by_object_id from core.meta import by_remote_id @@ -52,6 +53,16 @@ SIG_AUTH = HTTPSigAuth(KEY) MY_PERSON = ap.Person(**ME) +_LOCAL_NETLOC = urlparse(BASE_URL).netloc + + +def is_from_outbox(activity: ap.BaseActivity) -> bool: + return activity.id.startswith(BASE_URL) + + +def is_local_url(url: str) -> bool: + return urlparse(url).netloc == _LOCAL_NETLOC + def _remove_id(doc: ap.ObjectType) -> ap.ObjectType: """Helper for removing MongoDB's `_id` field.""" @@ -116,6 +127,11 @@ def save(box: Box, activity: ap.BaseActivity) -> None: actor_id = activity.get_actor().id + # Set some "type"-related neta + extra = {} + if box == Box.OUTBOX and activity.has_type(ap.Follow): + extra[MetaKey.FOLLOW_STATUS.value] = FollowStatus.WAITING.value + DB.activities.insert_one( { "box": box.value, @@ -135,6 +151,7 @@ def save(box: Box, activity: ap.BaseActivity) -> None: MetaKey.PUBLISHED.value: activity.published if activity.published else now(), + **extra, }, } ) diff --git a/core/inbox.py b/core/inbox.py index df5740b..939f2f0 100644 --- a/core/inbox.py +++ b/core/inbox.py @@ -9,10 +9,12 @@ from little_boxes.errors import NotAnActivityError import config from core.activitypub import _answer_key from core.activitypub import handle_replies +from core.activitypub import is_from_outbox from core.activitypub import post_to_outbox from core.activitypub import update_cached_actor from core.db import DB from core.db import update_one_activity +from core.meta import FollowStatus from core.meta import MetaKey from core.meta import by_object_id from core.meta import by_remote_id @@ -174,6 +176,33 @@ def _follow_process_inbox(activity: ap.Follow, new_meta: _NewMeta) -> None: post_to_outbox(accept) +def _update_follow_status(follow: ap.BaseActivity, status: FollowStatus) -> None: + if not follow.has_type(ap.Follow) or not is_from_outbox(follow): + _logger.warning( + "received an Accept/Reject from an unexpected activity: {follow!r}" + ) + return None + + update_one_activity( + by_remote_id(follow.id), upsert({MetaKey.FOLLOW_STATUS: status.value}) + ) + + +@process_inbox.register +def _accept_process_inbox(activity: ap.Accept, new_meta: _NewMeta) -> None: + _logger.info(f"process_inbox activity={activity!r}") + # Set a flag on the follow + follow = activity.get_object() + _update_follow_status(follow, FollowStatus.ACCEPTED) + + +@process_inbox.register +def _reject_process_inbox(activity: ap.Reject, new_meta: _NewMeta) -> None: + _logger.info(f"process_inbox activity={activity!r}") + follow = activity.get_object() + _update_follow_status(follow, FollowStatus.REJECTED) + + @process_inbox.register def _undo_process_inbox(activity: ap.Undo, new_meta: _NewMeta) -> None: _logger.info(f"process_inbox activity={activity!r}") diff --git a/core/meta.py b/core/meta.py index f5dde02..5e5a750 100644 --- a/core/meta.py +++ b/core/meta.py @@ -18,6 +18,13 @@ class Box(Enum): REPLIES = "replies" +@unique +class FollowStatus(Enum): + WAITING = "waiting" + ACCEPTED = "accepted" + REJECTED = "rejected" + + @unique class MetaKey(Enum): NOTIFICATION = "notification" @@ -38,6 +45,9 @@ class MetaKey(Enum): OBJECT_ACTOR_ID = "object_actor_id" OBJECT_ACTOR_HASH = "object_actor_hash" PUBLIC = "public" + + FOLLOW_STATUS = "follow_status" + THREAD_ROOT_PARENT = "thread_root_parent" IN_REPLY_TO_SELF = "in_reply_to_self" @@ -83,6 +93,10 @@ def by_type(type_: Union[ap.ActivityType, List[ap.ActivityType]]) -> _SubQuery: return {"type": type_.value} +def follow_request_accepted() -> _SubQuery: + return flag(MetaKey.FOLLOW_STATUS, FollowStatus.ACCEPTED.value) + + def not_undo() -> _SubQuery: return flag(MetaKey.UNDO, False) @@ -95,6 +109,10 @@ def by_actor(actor: ap.BaseActivity) -> _SubQuery: return flag(MetaKey.ACTOR_ID, actor.id) +def by_actor_id(actor_id: str) -> _SubQuery: + return flag(MetaKey.ACTOR_ID, actor_id) + + def by_object_id(object_id: str) -> _SubQuery: return flag(MetaKey.OBJECT_ID, object_id) diff --git a/core/migrations.py b/core/migrations.py index 7c9d727..9f20a9d 100644 --- a/core/migrations.py +++ b/core/migrations.py @@ -10,8 +10,16 @@ from core import activitypub from core.db import DB from core.db import find_activities from core.db import update_one_activity +from core.meta import FollowStatus from core.meta import MetaKey from core.meta import _meta +from core.meta import by_actor_id +from core.meta import by_actor +from core.meta import by_remote_id +from core.meta import by_type +from core.meta import in_inbox +from core.meta import not_undo +from core.meta import upsert from utils.migrations import Migration from utils.migrations import logger from utils.migrations import perform # noqa: just here for export @@ -156,10 +164,10 @@ class _2_FollowMigration(Migration): {"_id": data["_id"]}, {"$set": {"meta.actor": actor}} ) except Exception: - logger.exception("failed to process actor {data!r}") + logger.exception(f"failed to process actor {data!r}") -class _20190808_MetaPublishedMigration(Migration): +class _20190830_MetaPublishedMigration(Migration): """Add the `meta.published` field to old activities.""" def migrate(self) -> None: @@ -180,4 +188,68 @@ class _20190808_MetaPublishedMigration(Migration): ) except Exception: - logger.exception("failed to process activity {data!r}") + logger.exception(f"failed to process activity {data!r}") + + +class _20190830_FollowFollowBackMigration(Migration): + """Add the new meta flags for tracking accepted/rejected status and following/follows back info.""" + + def migrate(self) -> None: + for data in find_activities({**by_type(ap.ActivityType.ACCEPT), **in_inbox()}): + try: + update_one_activity( + {**by_type(ap.ActivityType.FOLLOW), **by_remote_id(data["meta"]["object_id"])}, + upsert({MetaKey.FOLLOW_STATUS: FollowStatus.ACCEPTED.value}), + ) + # Check if we are following this actor + follow_query = { + **in_inbox(), + **by_type(ap.ActivityType.FOLLOW), + **by_actor_id(data["meta"]["actor_id"]), + **not_undo(), + } + raw_follow = DB.activities.find_one(follow_query) + if raw_follow: + DB.activities.update_many( + follow_query, + {"$set": {_meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True}}, + ) + + except Exception: + logger.exception(f"failed to process activity {data!r}") + + for data in find_activities({**by_type(ap.ActivityType.REJECT), **in_inbox()}): + try: + update_one_activity( + {**by_type(ap.ActivityType.FOLLOW), **by_remote_id(data["meta"]["object_id"])}, + upsert({MetaKey.FOLLOW_STATUS: FollowStatus.REJECTED.value}), + ) + except Exception: + logger.exception(f"failed to process activity {data!r}") + + for data in find_activities({**by_type(ap.ActivityType.FOLLOW), **in_inbox()}): + try: + accept_query = { + **in_inbox(), + **by_type(ap.ActivityType.ACCEPT), + **by_actor_id(data["meta"]["actor_id"]), + **not_undo(), + } + raw_accept = DB.activities.find_one(accept_query) + if raw_accept: + DB.activities.update_many( + accept_query, + {"$set": {_meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True}}, + ) + + except Exception: + logger.exception(f"failed to process activity {data!r}") + + DB.activities.update_many( + { + **by_type(ap.ActivityType.FOLLOW), + **in_inbox(), + "meta.follow_status": {"$exists": False}, + }, + {"$set": {"meta.follow_status": "waiting"}}, + ) diff --git a/core/notifications.py b/core/notifications.py index 1710a85..dfb6d7d 100644 --- a/core/notifications.py +++ b/core/notifications.py @@ -5,12 +5,12 @@ from datetime import timezone from functools import singledispatch from typing import Any from typing import Dict -from urllib.parse import urlparse from little_boxes import activitypub as ap -from config import BASE_URL from config import DB +from core.activitypub import is_from_outbox +from core.activitypub import is_local_url from core.db import find_one_activity from core.meta import MetaKey from core.meta import _meta @@ -27,16 +27,6 @@ _logger = logging.getLogger(__name__) _NewMeta = Dict[str, Any] -_LOCAL_NETLOC = urlparse(BASE_URL).netloc - - -def _is_from_outbox(activity: ap.BaseActivity) -> bool: - return activity.id.startswith(BASE_URL) - - -def _is_local(url: str) -> bool: - return urlparse(url).netloc == _LOCAL_NETLOC - def _flag_as_notification(activity: ap.BaseActivity, new_meta: _NewMeta) -> None: new_meta.update( @@ -83,6 +73,16 @@ def _accept_set_inbox_flags(activity: ap.Accept, new_meta: _NewMeta) -> None: return None +@set_inbox_flags.register +def _reject_set_inbox_flags(activity: ap.Reject, new_meta: _NewMeta) -> None: + """Handle notifications for "rejected" following requests.""" + _logger.info(f"set_inbox_flags activity={activity!r}") + # This Accept will be a "You started following $actor" notification + _flag_as_notification(activity, new_meta) + _set_flag(new_meta, MetaKey.GC_KEEP) + return None + + @set_inbox_flags.register def _follow_set_inbox_flags(activity: ap.Follow, new_meta: _NewMeta) -> None: """Handle notification for new followers.""" @@ -114,7 +114,7 @@ def _follow_set_inbox_flags(activity: ap.Follow, new_meta: _NewMeta) -> None: def _like_set_inbox_flags(activity: ap.Like, new_meta: _NewMeta) -> None: _logger.info(f"set_inbox_flags activity={activity!r}") # Is it a Like of local acitivty/from the outbox - if _is_from_outbox(activity.get_object()): + if is_from_outbox(activity.get_object()): # Flag it as a notification _flag_as_notification(activity, new_meta) @@ -132,7 +132,7 @@ def _announce_set_inbox_flags(activity: ap.Announce, new_meta: _NewMeta) -> None _logger.info(f"set_inbox_flags activity={activity!r}") obj = activity.get_object() # Is it a Annnounce/boost of local acitivty/from the outbox - if _is_from_outbox(obj): + if is_from_outbox(obj): # Flag it as a notification _flag_as_notification(activity, new_meta) @@ -180,7 +180,7 @@ def _create_set_inbox_flags(activity: ap.Create, new_meta: _NewMeta) -> None: in_reply_to = obj.get_in_reply_to() # Check if it's a local reply - if in_reply_to and _is_local(in_reply_to): + if in_reply_to and is_local_url(in_reply_to): # TODO(tsileo): fetch the reply to check for poll answers more precisely # reply_of = ap.fetch_remote_activity(in_reply_to) @@ -199,7 +199,7 @@ def _create_set_inbox_flags(activity: ap.Create, new_meta: _NewMeta) -> None: # Check for mention for mention in obj.get_mentions(): - if mention.href and _is_local(mention.href): + if mention.href and is_local_url(mention.href): # Flag it as a notification _flag_as_notification(activity, new_meta) From 795ef0990cf4dea32ab3abcceb98a7654c7a633e Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Sep 2019 10:34:02 +0200 Subject: [PATCH 0667/1425] Fix formatting --- core/migrations.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/core/migrations.py b/core/migrations.py index 9f20a9d..970e543 100644 --- a/core/migrations.py +++ b/core/migrations.py @@ -198,7 +198,10 @@ class _20190830_FollowFollowBackMigration(Migration): for data in find_activities({**by_type(ap.ActivityType.ACCEPT), **in_inbox()}): try: update_one_activity( - {**by_type(ap.ActivityType.FOLLOW), **by_remote_id(data["meta"]["object_id"])}, + { + **by_type(ap.ActivityType.FOLLOW), + **by_remote_id(data["meta"]["object_id"]), + }, upsert({MetaKey.FOLLOW_STATUS: FollowStatus.ACCEPTED.value}), ) # Check if we are following this actor @@ -221,7 +224,10 @@ class _20190830_FollowFollowBackMigration(Migration): for data in find_activities({**by_type(ap.ActivityType.REJECT), **in_inbox()}): try: update_one_activity( - {**by_type(ap.ActivityType.FOLLOW), **by_remote_id(data["meta"]["object_id"])}, + { + **by_type(ap.ActivityType.FOLLOW), + **by_remote_id(data["meta"]["object_id"]), + }, upsert({MetaKey.FOLLOW_STATUS: FollowStatus.REJECTED.value}), ) except Exception: From 5f9c6b8dad5f7341617988a6455685d40163cf02 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Sep 2019 10:37:05 +0200 Subject: [PATCH 0668/1425] Remove unused import --- core/migrations.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/migrations.py b/core/migrations.py index 970e543..49cf534 100644 --- a/core/migrations.py +++ b/core/migrations.py @@ -14,7 +14,6 @@ from core.meta import FollowStatus from core.meta import MetaKey from core.meta import _meta from core.meta import by_actor_id -from core.meta import by_actor from core.meta import by_remote_id from core.meta import by_type from core.meta import in_inbox From 96121e57f30c44f2604d1c277cb7dc9b91537b16 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Sep 2019 10:45:18 +0200 Subject: [PATCH 0669/1425] Fix BeautifulSoup warning --- utils/highlight.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/highlight.py b/utils/highlight.py index 74fa16f..66352b4 100644 --- a/utils/highlight.py +++ b/utils/highlight.py @@ -22,7 +22,7 @@ def highlight(html: str) -> str: if not code.parent.name == "pre": continue lexer = guess_lexer(code.text) - tag = BeautifulSoup(phighlight(code.text, lexer, _FORMATTER)).body.next + tag = BeautifulSoup(phighlight(code.text, lexer, _FORMATTER), "html5lib").body.next pre = code.parent pre.replaceWith(tag) out = soup.body From c530b24589da705c97260427337e82b78ae5fd90 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Sep 2019 10:48:02 +0200 Subject: [PATCH 0670/1425] Fix internal profile page links --- blueprints/admin.py | 12 +++++++++++- templates/stream.html | 2 +- utils/highlight.py | 4 +++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/blueprints/admin.py b/blueprints/admin.py index c304a46..4cd835c 100644 --- a/blueprints/admin.py +++ b/blueprints/admin.py @@ -26,6 +26,10 @@ from config import PASS from core.activitypub import Box from core.activitypub import post_to_outbox from core.db import find_one_activity +from core.meta import in_outbox +from core.meta import by_actor +from core.meta import not_undo +from core.meta import follow_request_accepted from core.meta import by_object_id from core.meta import by_type from core.shared import MY_PERSON @@ -233,7 +237,13 @@ def admin_profile() -> _Response: } ) following = find_one_activity( - {"type": ap.ActivityType.ACCEPT.value, "meta.actor_id": actor.id} + { + **by_type(ap.ActivityType.FOLLOW), + **by_object_id(actor.id), + **not_undo(), + **in_outbox(), + **follow_request_accepted(), + } ) return htmlify( diff --git a/templates/stream.html b/templates/stream.html index a9736b8..163654c 100644 --- a/templates/stream.html +++ b/templates/stream.html @@ -27,7 +27,7 @@ {% if following %} - + diff --git a/utils/highlight.py b/utils/highlight.py index 66352b4..074108d 100644 --- a/utils/highlight.py +++ b/utils/highlight.py @@ -22,7 +22,9 @@ def highlight(html: str) -> str: if not code.parent.name == "pre": continue lexer = guess_lexer(code.text) - tag = BeautifulSoup(phighlight(code.text, lexer, _FORMATTER), "html5lib").body.next + tag = BeautifulSoup( + phighlight(code.text, lexer, _FORMATTER), "html5lib" + ).body.next pre = code.parent pre.replaceWith(tag) out = soup.body From bb60ac866222ec07cf01d1ea5a58b5d40ccf4847 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Sep 2019 10:49:09 +0200 Subject: [PATCH 0671/1425] Fix docker conf --- .dockerignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index f24c558..fdf7aa9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,3 @@ -.git/ __pycache__/ data/ data2/ From efc8a416240c3004328b89eb34cde1ceeaa647d2 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Sep 2019 10:51:45 +0200 Subject: [PATCH 0672/1425] Cleanup imports --- blueprints/admin.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/blueprints/admin.py b/blueprints/admin.py index 4cd835c..b07ff7c 100644 --- a/blueprints/admin.py +++ b/blueprints/admin.py @@ -26,12 +26,11 @@ from config import PASS from core.activitypub import Box from core.activitypub import post_to_outbox from core.db import find_one_activity -from core.meta import in_outbox -from core.meta import by_actor -from core.meta import not_undo -from core.meta import follow_request_accepted from core.meta import by_object_id from core.meta import by_type +from core.meta import follow_request_accepted +from core.meta import in_outbox +from core.meta import not_undo from core.shared import MY_PERSON from core.shared import _build_thread from core.shared import _Response From 1ab8920df822d3312104da02a02db82a8e42a62b Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Sep 2019 11:32:12 +0200 Subject: [PATCH 0673/1425] More following/follows back stuff --- app.py | 6 ++-- core/migrations.py | 71 ++++++++++++++++++++++++++++++---------- templates/followers.html | 8 ++++- templates/following.html | 7 +++- 4 files changed, 68 insertions(+), 24 deletions(-) diff --git a/app.py b/app.py index bf2f838..e3e17a3 100644 --- a/app.py +++ b/app.py @@ -818,9 +818,7 @@ def followers(): ) raw_followers, older_than, newer_than = paginated_query(DB.activities, q) - followers = [ - doc["meta"]["actor"] for doc in raw_followers if "actor" in doc.get("meta", {}) - ] + followers = [doc["meta"] for doc in raw_followers if "actor" in doc.get("meta", {})] return htmlify( render_template( "followers.html", @@ -857,7 +855,7 @@ def following(): following, older_than, newer_than = paginated_query(DB.activities, q) following = [ - (doc["remote_id"], doc["meta"]["object"]) + (doc["remote_id"], doc["meta"]) for doc in following if "remote_id" in doc and "object" in doc.get("meta", {}) ] diff --git a/core/migrations.py b/core/migrations.py index 49cf534..8d7bcd4 100644 --- a/core/migrations.py +++ b/core/migrations.py @@ -14,9 +14,11 @@ from core.meta import FollowStatus from core.meta import MetaKey from core.meta import _meta from core.meta import by_actor_id +from core.meta import by_object_id from core.meta import by_remote_id from core.meta import by_type from core.meta import in_inbox +from core.meta import in_outbox from core.meta import not_undo from core.meta import upsert from utils.migrations import Migration @@ -232,24 +234,6 @@ class _20190830_FollowFollowBackMigration(Migration): except Exception: logger.exception(f"failed to process activity {data!r}") - for data in find_activities({**by_type(ap.ActivityType.FOLLOW), **in_inbox()}): - try: - accept_query = { - **in_inbox(), - **by_type(ap.ActivityType.ACCEPT), - **by_actor_id(data["meta"]["actor_id"]), - **not_undo(), - } - raw_accept = DB.activities.find_one(accept_query) - if raw_accept: - DB.activities.update_many( - accept_query, - {"$set": {_meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True}}, - ) - - except Exception: - logger.exception(f"failed to process activity {data!r}") - DB.activities.update_many( { **by_type(ap.ActivityType.FOLLOW), @@ -258,3 +242,54 @@ class _20190830_FollowFollowBackMigration(Migration): }, {"$set": {"meta.follow_status": "waiting"}}, ) + + +class _20190901_FollowFollowBackMigrationFix(Migration): + """Add the new meta flags for tracking accepted/rejected status and following/follows back info.""" + + def migrate(self) -> None: + for data in find_activities({**by_type(ap.ActivityType.ACCEPT), **in_inbox()}): + try: + update_one_activity( + { + **by_type(ap.ActivityType.FOLLOW), + **by_remote_id(data["meta"]["object_id"]), + }, + upsert({MetaKey.FOLLOW_STATUS: FollowStatus.ACCEPTED.value}), + ) + # Check if we are following this actor + follow_query = { + **in_inbox(), + **by_type(ap.ActivityType.FOLLOW), + **by_object_id(data["meta"]["actor_id"]), + **not_undo(), + } + raw_follow = DB.activities.find_one(follow_query) + if raw_follow: + DB.activities.update_many( + follow_query, + {"$set": {_meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True}}, + ) + + except Exception: + logger.exception(f"failed to process activity {data!r}") + + for data in find_activities({**by_type(ap.ActivityType.FOLLOW), **in_outbox()}): + try: + print(data) + follow_query = { + **in_inbox(), + **by_type(ap.ActivityType.FOLLOW), + **by_actor_id(data["meta"]["object_id"]), + **not_undo(), + } + raw_accept = DB.activities.find_one(follow_query) + print(raw_accept) + if raw_accept: + DB.activities.update_many( + by_remote_id(data["remote_id"]), + {"$set": {_meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True}}, + ) + + except Exception: + logger.exception(f"failed to process activity {data!r}") diff --git a/templates/followers.html b/templates/followers.html index f1bba01..c01f880 100644 --- a/templates/followers.html +++ b/templates/followers.html @@ -8,10 +8,16 @@ {% include "header.html" %}
    - {% for follower in followers_data %} + {% for meta in followers_data %} + {% set follower = meta.actor %} {% if session.logged_in %}
    profile + +{% if meta.notification_follows_back %} +following +{% endif %} +
    {% endif %}
    diff --git a/templates/following.html b/templates/following.html index f084d32..bdb5361 100644 --- a/templates/following.html +++ b/templates/following.html @@ -8,7 +8,8 @@ {% include "header.html" %}
    - {% for (follow_id, follow) in following_data %} + {% for (follow_id, meta) in following_data %} + {% set follow = meta.object %} {% if session.logged_in %}
    profile @@ -18,6 +19,9 @@ +{% if meta.notification_follows_back %} +follows you back +{% endif %} {% if lists %}
    @@ -48,6 +52,7 @@ {% endif %} {% endfor %} +
    From 3a141795e6860ea2f6febf9cbe4ee4f01ea6c795 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Sep 2019 14:19:33 +0200 Subject: [PATCH 0674/1425] Fix template and improve threads --- core/shared.py | 4 +++- templates/utils.html | 28 ++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/core/shared.py b/core/shared.py index e1ab120..41c5192 100644 --- a/core/shared.py +++ b/core/shared.py @@ -142,7 +142,9 @@ def _get_ip(): def _build_thread(data, include_children=True): # noqa: C901 data["_requested"] = True app.logger.info(f"_build_thread({data!r})") - root_id = data["meta"][MetaKey.OBJECT_ID.value] + root_id = data["meta"].get( + MetaKey.THREAD_ROOT_PARENT.value, data["meta"][MetaKey.OBJECT_ID.value] + ) replies = [data] for dat in find_activities( diff --git a/templates/utils.html b/templates/utils.html index 34b5eb0..ed49404 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -15,6 +15,17 @@ {% endif %} {%- endmacro %} +{% macro display_actor_oneline(follower) -%} +{% if follower and follower.id %} + + +{{ (follower.name or follower.preferredUsername) | clean | replace_custom_emojis(follower) | safe }} +@{{ follower.preferredUsername }}@{{ follower | url_or_id | get_url | domain }} + + +{% endif %} +{%- endmacro %} + {% macro display_note(obj, perma=False, ui=False, likes=[], shares=[], meta={}, no_color=False) -%} {% if meta.object_actor %} @@ -60,11 +71,22 @@ {% if not perma %} - + {% endif %}
    +{% if meta.in_reply_to_actor %} +
    +in reply to {{ display_actor_oneline(meta.in_reply_to_actor) }} +
    +{% elif meta.in_reply_to_self %} +
    +self reply +
    +{% endif %} + + {% if obj.summary %}

    {{ obj.summary | clean | replace_custom_emojis(obj) | safe }}

    {% endif %} {% if obj | has_type('Video') %}
    @@ -321,7 +343,9 @@ {% endif %} {% if meta.object_visibility | visibility_is_public %} -{% if obj | url_or_id | is_from_outbox %} +{% if obj.inReplyTo and not meta.count_reply and not perma %} +thread{% endif %} +{% if obj | url_or_id | get_url | is_from_outbox %} permalink {% else %} source From 36bc93cfda275464e5afcd27d7baea98572a365f Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Sep 2019 14:28:50 +0200 Subject: [PATCH 0675/1425] Fix the thread button --- templates/utils.html | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/templates/utils.html b/templates/utils.html index ed49404..32683fa 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -342,9 +342,11 @@ self reply {% endif %} +{% if session.logged_in and obj.inReplyTo and not meta.count_reply and not perma %} +thread +{% endif %} + {% if meta.object_visibility | visibility_is_public %} -{% if obj.inReplyTo and not meta.count_reply and not perma %} -thread{% endif %} {% if obj | url_or_id | get_url | is_from_outbox %} permalink {% else %} From 52bc600832ad11302c9b486473be2eff8a4cc41f Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Sep 2019 20:58:51 +0200 Subject: [PATCH 0676/1425] Index hashtags and mentions --- app.py | 29 +++++++++++++++-------------- core/activitypub.py | 15 +++++++++++++-- core/indexes.py | 2 ++ core/meta.py | 11 +++++++++++ core/migrations.py | 25 +++++++++++++++++++++++++ 5 files changed, 66 insertions(+), 16 deletions(-) diff --git a/app.py b/app.py index e3e17a3..2940d42 100644 --- a/app.py +++ b/app.py @@ -55,10 +55,13 @@ from core.db import find_one_activity from core.meta import Box from core.meta import MetaKey from core.meta import _meta +from core.meta import by_hashtag from core.meta import by_remote_id from core.meta import by_type +from core.meta import by_visibility from core.meta import in_outbox from core.meta import is_public +from core.meta import not_deleted from core.meta import not_undo from core.shared import _build_thread from core.shared import _get_ip @@ -875,9 +878,10 @@ def following(): def tags(tag): if not DB.activities.count( { - "box": Box.OUTBOX.value, - "activity.object.tag.type": "Hashtag", - "activity.object.tag.name": "#" + tag, + **in_outbox(), + **by_hashtag(tag), + **by_visibility(ap.Visibility.PUBLIC), + **not_deleted(), } ): abort(404) @@ -888,23 +892,20 @@ def tags(tag): tag=tag, outbox_data=DB.activities.find( { - "box": Box.OUTBOX.value, - "type": ActivityType.CREATE.value, - "meta.deleted": False, - "activity.object.tag.type": "Hashtag", - "activity.object.tag.name": "#" + tag, + **in_outbox(), + **by_hashtag(tag), + **by_visibility(ap.Visibility.PUBLIC), + **not_deleted(), } ), ) ) _log_sig() q = { - "box": Box.OUTBOX.value, - "meta.deleted": False, - "meta.undo": False, - "type": ActivityType.CREATE.value, - "activity.object.tag.type": "Hashtag", - "activity.object.tag.name": "#" + tag, + **in_outbox(), + **by_hashtag(tag), + **by_visibility(ap.Visibility.PUBLIC), + **not_deleted(), } return activitypubify( **activitypub.build_ordered_collection( diff --git a/core/activitypub.py b/core/activitypub.py index 65e488c..4522484 100644 --- a/core/activitypub.py +++ b/core/activitypub.py @@ -128,9 +128,20 @@ def save(box: Box, activity: ap.BaseActivity) -> None: actor_id = activity.get_actor().id # Set some "type"-related neta - extra = {} - if box == Box.OUTBOX and activity.has_type(ap.Follow): + extra: Dict[str, Any] = {} + if box == Box.OUTBOX and activity.has_type(ap.ActivityType.FOLLOW): extra[MetaKey.FOLLOW_STATUS.value] = FollowStatus.WAITING.value + elif activity.has_type(ap.ActivityType.CREATE): + mentions = [] + obj = activity.get_object() + for m in obj.get_mentions(): + mentions.append(m.href) + hashtags = [] + for h in obj.get_hashtags(): + hashtags.append(h.name[1:]) # Strip the # + extra.update( + {MetaKey.MENTIONS.value: mentions, MetaKey.HASHTAGS.value: hashtags} + ) DB.activities.insert_one( { diff --git a/core/indexes.py b/core/indexes.py index a78ba02..9df3485 100644 --- a/core/indexes.py +++ b/core/indexes.py @@ -26,6 +26,8 @@ def create_indexes(): DB.activities.create_index([("remote_id", pymongo.ASCENDING)]) DB.activities.create_index([("meta.actor_id", pymongo.ASCENDING)]) DB.activities.create_index([("meta.object_id", pymongo.ASCENDING)]) + DB.activities.create_index([("meta.mentions", pymongo.ASCENDING)]) + DB.activities.create_index([("meta.hashtags", pymongo.ASCENDING)]) DB.activities.create_index([("meta.thread_root_parent", pymongo.ASCENDING)]) DB.activities.create_index( [ diff --git a/core/meta.py b/core/meta.py index 5e5a750..519ebbf 100644 --- a/core/meta.py +++ b/core/meta.py @@ -46,6 +46,9 @@ class MetaKey(Enum): OBJECT_ACTOR_HASH = "object_actor_hash" PUBLIC = "public" + HASHTAGS = "hashtags" + MENTIONS = "mentions" + FOLLOW_STATUS = "follow_status" THREAD_ROOT_PARENT = "thread_root_parent" @@ -121,6 +124,14 @@ def is_public() -> _SubQuery: return flag(MetaKey.PUBLIC, True) +def by_visibility(vis: ap.Visibility) -> _SubQuery: + return flag(MetaKey.VISIBILITY, vis.name) + + +def by_hashtag(ht: str) -> _SubQuery: + return flag(MetaKey.HASHTAGS, ht) + + def inc(mk: MetaKey, val: int) -> _SubQuery: return {"$inc": flag(mk, val)} diff --git a/core/migrations.py b/core/migrations.py index 8d7bcd4..db39438 100644 --- a/core/migrations.py +++ b/core/migrations.py @@ -19,6 +19,7 @@ from core.meta import by_remote_id from core.meta import by_type from core.meta import in_inbox from core.meta import in_outbox +from core.meta import not_deleted from core.meta import not_undo from core.meta import upsert from utils.migrations import Migration @@ -293,3 +294,27 @@ class _20190901_FollowFollowBackMigrationFix(Migration): except Exception: logger.exception(f"failed to process activity {data!r}") + + +class _20190901_MetaHashtagsAndMentions(Migration): + def migrate(self) -> None: + for data in find_activities( + {**by_type(ap.ActivityType.CREATE), **not_deleted()} + ): + try: + activity = ap.parse_activity(data["activity"]) + mentions = [] + obj = activity.get_object() + for m in obj.get_mentions(): + mentions.append(m.href) + hashtags = [] + for h in obj.get_hashtags(): + hashtags.append(h.name[1:]) # Strip the # + + update_one_activity( + by_remote_id(data["remote_id"]), + upsert({MetaKey.MENTIONS: mentions, MetaKey.HASHTAGS: hashtags}), + ) + + except Exception: + logger.exception(f"failed to process activity {data!r}") From 5b1e776fa28979d28bc21cefc6b00c8e523b92b4 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Sep 2019 21:38:38 +0200 Subject: [PATCH 0677/1425] Visibility tweaks --- app.py | 53 ++++++++++++++++++++++++---------------------------- core/meta.py | 9 +++++++++ 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/app.py b/app.py index 2940d42..33f0847 100644 --- a/app.py +++ b/app.py @@ -63,6 +63,7 @@ from core.meta import in_outbox from core.meta import is_public from core.meta import not_deleted from core.meta import not_undo +from core.meta import pinned from core.shared import _build_thread from core.shared import _get_ip from core.shared import activitypubify @@ -107,14 +108,12 @@ else: @app.context_processor def inject_config(): q = { - "type": "Create", - "activity.object.inReplyTo": None, - "meta.deleted": False, - "meta.public": True, + **in_outbox(), + **by_type([ActivityType.CREATE, ActivityType.ANNOUNCE]), + **not_deleted(), + **by_visibility(ap.Visibility.PUBLIC), } - notes_count = DB.activities.find( - {"box": Box.OUTBOX.value, "$or": [q, {"type": "Announce", "meta.undo": False}]} - ).count() + notes_count = DB.activities.count(q) # FIXME(tsileo): rename to all_count, and remove poll answers from it all_q = { "box": Box.OUTBOX.value, @@ -348,30 +347,27 @@ def index(): return activitypubify(**ME) q = { - "box": Box.OUTBOX.value, - "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, - "activity.object.inReplyTo": None, - "meta.deleted": False, - "meta.undo": False, - "meta.public": True, + **in_outbox(), + **by_type([ActivityType.CREATE, ActivityType.ANNOUNCE]), + **not_deleted(), + **by_visibility(ap.Visibility.PUBLIC), "$or": [{"meta.pinned": False}, {"meta.pinned": {"$exists": False}}], } - pinned = [] + apinned = [] # Only fetch the pinned notes if we're on the first page if not request.args.get("older_than") and not request.args.get("newer_than"): q_pinned = { - "box": Box.OUTBOX.value, - "type": ActivityType.CREATE.value, - "meta.deleted": False, - "meta.undo": False, - "meta.public": True, - "meta.pinned": True, + **in_outbox(), + **by_type(ActivityType.CREATE), + **not_deleted(), + **pinned(), + **by_visibility(ap.Visibility.PUBLIC), } - pinned = list(DB.activities.find(q_pinned)) + apinned = list(DB.activities.find(q_pinned)) outbox_data, older_than, newer_than = paginated_query( - DB.activities, q, limit=25 - len(pinned) + DB.activities, q, limit=25 - len(apinned) ) return htmlify( @@ -380,7 +376,7 @@ def index(): outbox_data=outbox_data, older_than=older_than, newer_than=newer_than, - pinned=pinned, + pinned=apinned, ) ) @@ -481,11 +477,10 @@ def outbox(): _log_sig() # TODO(tsileo): returns the whole outbox if authenticated and look at OCAP support q = { - "box": Box.OUTBOX.value, - "meta.deleted": False, - "meta.undo": False, - "meta.public": True, - "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, + **in_outbox(), + **by_type([ActivityType.CREATE, ActivityType.ANNOUNCE]), + **not_deleted(), + **by_visibility(ap.Visibility.PUBLIC), } return activitypubify( **activitypub.build_ordered_collection( @@ -497,7 +492,7 @@ def outbox(): ) ) - # Handle POST request + # Handle POST request aka C2S API try: _api_required() except BadSignature: diff --git a/core/meta.py b/core/meta.py index 519ebbf..40215dd 100644 --- a/core/meta.py +++ b/core/meta.py @@ -46,6 +46,7 @@ class MetaKey(Enum): OBJECT_ACTOR_HASH = "object_actor_hash" PUBLIC = "public" + PINNED = "pinned" HASHTAGS = "hashtags" MENTIONS = "mentions" @@ -100,6 +101,10 @@ def follow_request_accepted() -> _SubQuery: return flag(MetaKey.FOLLOW_STATUS, FollowStatus.ACCEPTED.value) +def not_in_reply_to() -> _SubQuery: + return {"activity.object.inReplyTo": None} + + def not_undo() -> _SubQuery: return flag(MetaKey.UNDO, False) @@ -108,6 +113,10 @@ def not_deleted() -> _SubQuery: return flag(MetaKey.DELETED, False) +def pinned() -> _SubQuery: + return flag(MetaKey.PINNED, True) + + def by_actor(actor: ap.BaseActivity) -> _SubQuery: return flag(MetaKey.ACTOR_ID, actor.id) From 151ced0b417de415f2ce73fdfe5c456b14c41dea Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 1 Sep 2019 21:49:36 +0200 Subject: [PATCH 0678/1425] Fix queries for index/outbox --- app.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/app.py b/app.py index 33f0847..194d492 100644 --- a/app.py +++ b/app.py @@ -109,9 +109,14 @@ else: def inject_config(): q = { **in_outbox(), - **by_type([ActivityType.CREATE, ActivityType.ANNOUNCE]), - **not_deleted(), - **by_visibility(ap.Visibility.PUBLIC), + "$or": [ + { + **by_type(ActivityType.CREATE), + **not_deleted(), + **by_visibility(ap.Visibility.PUBLIC), + }, + {**by_type(ActivityType.ANNOUNCE), **not_undo()}, + ], } notes_count = DB.activities.count(q) # FIXME(tsileo): rename to all_count, and remove poll answers from it @@ -348,10 +353,15 @@ def index(): q = { **in_outbox(), - **by_type([ActivityType.CREATE, ActivityType.ANNOUNCE]), - **not_deleted(), - **by_visibility(ap.Visibility.PUBLIC), - "$or": [{"meta.pinned": False}, {"meta.pinned": {"$exists": False}}], + "$or": [ + { + **by_type(ActivityType.CREATE), + **not_deleted(), + **by_visibility(ap.Visibility.PUBLIC), + "$or": [{"meta.pinned": False}, {"meta.pinned": {"$exists": False}}], + }, + {**by_type(ActivityType.ANNOUNCE), **not_undo()}, + ], } apinned = [] @@ -478,9 +488,14 @@ def outbox(): # TODO(tsileo): returns the whole outbox if authenticated and look at OCAP support q = { **in_outbox(), - **by_type([ActivityType.CREATE, ActivityType.ANNOUNCE]), - **not_deleted(), - **by_visibility(ap.Visibility.PUBLIC), + "$or": [ + { + **by_type(ActivityType.CREATE), + **not_deleted(), + **by_visibility(ap.Visibility.PUBLIC), + }, + {**by_type(ActivityType.ANNOUNCE), **not_undo()}, + ], } return activitypubify( **activitypub.build_ordered_collection( From bcec7146fd8c334ff7bb25dea7f1a48cc28e59bb Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Mon, 2 Sep 2019 23:44:38 +0200 Subject: [PATCH 0679/1425] Cleanup queries --- app.py | 89 ++++++++++++++++++++++++++-------------------------- core/meta.py | 4 +++ 2 files changed, 48 insertions(+), 45 deletions(-) diff --git a/app.py b/app.py index 194d492..f636f01 100644 --- a/app.py +++ b/app.py @@ -56,12 +56,15 @@ from core.meta import Box from core.meta import MetaKey from core.meta import _meta from core.meta import by_hashtag +from core.meta import by_object_id from core.meta import by_remote_id from core.meta import by_type from core.meta import by_visibility +from core.meta import in_inbox from core.meta import in_outbox from core.meta import is_public from core.meta import not_deleted +from core.meta import not_poll_answer from core.meta import not_undo from core.meta import pinned from core.shared import _build_thread @@ -121,27 +124,29 @@ def inject_config(): notes_count = DB.activities.count(q) # FIXME(tsileo): rename to all_count, and remove poll answers from it all_q = { - "box": Box.OUTBOX.value, - "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, - "meta.undo": False, - "meta.deleted": False, - "meta.poll_answer": False, + **in_outbox(), + **by_type([ActivityType.CREATE, ActivityType.ANNOUNCE]), + **not_deleted(), + **not_undo(), + **not_poll_answer(), } liked_q = { **in_outbox(), - "meta.deleted": False, - "meta.undo": False, - "type": ActivityType.LIKE.value, + **by_type(ActivityType.LIKE), + **not_undo(), + **not_deleted(), } followers_q = { - "box": Box.INBOX.value, - "type": ActivityType.FOLLOW.value, - "meta.undo": False, + **in_inbox(), + **by_type(ActivityType.FOLLOW), + **not_undo(), + **not_deleted(), } following_q = { - "box": Box.OUTBOX.value, - "type": ActivityType.FOLLOW.value, - "meta.undo": False, + **in_outbox(), + **by_type(ActivityType.FOLLOW), + **not_undo(), + **not_deleted(), } unread_notifications_q = {_meta(MetaKey.NOTIFICATION_UNREAD): True} @@ -395,11 +400,11 @@ def index(): @login_required def all(): q = { - "box": Box.OUTBOX.value, - "type": {"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]}, - "meta.deleted": False, - "meta.undo": False, - "meta.poll_answer": False, + **in_outbox(), + **by_type([ActivityType.CREATE, ActivityType.ANNOUNCE]), + **not_deleted(), + **not_undo(), + **not_poll_answer(), } outbox_data, older_than, newer_than = paginated_query(DB.activities, q) @@ -419,7 +424,7 @@ def note_by_id(note_id): return redirect(url_for("outbox_activity", item_id=note_id)) data = DB.activities.find_one( - {"box": Box.OUTBOX.value, "remote_id": activity_url(note_id)} + {**in_outbox(), **by_remote_id(activity_url(note_id))} ) if not data: abort(404) @@ -432,14 +437,10 @@ def note_by_id(note_id): raw_likes = list( DB.activities.find( { - "meta.undo": False, - "meta.deleted": False, - "type": ActivityType.LIKE.value, - "$or": [ - # FIXME(tsileo): remove all the useless $or - {"activity.object.id": data["activity"]["object"]["id"]}, - {"activity.object": data["activity"]["object"]["id"]}, - ], + **not_undo(), + **not_deleted(), + **by_type(ActivityType.LIKE), + **by_object_id(data["activity"]["object"]["id"]), } ) ) @@ -454,13 +455,10 @@ def note_by_id(note_id): raw_shares = list( DB.activities.find( { - "meta.undo": False, - "meta.deleted": False, - "type": ActivityType.ANNOUNCE.value, - "$or": [ - {"activity.object.id": data["activity"]["object"]["id"]}, - {"activity.object": data["activity"]["object"]["id"]}, - ], + **not_undo(), + **not_deleted(), + **by_type(ActivityType.ANNOUNCE), + **by_object_id(data["activity"]["object"]["id"]), } ) ) @@ -531,9 +529,10 @@ def ap_emoji(name): def outbox_detail(item_id): doc = DB.activities.find_one( { - "box": Box.OUTBOX.value, - "remote_id": activity_url(item_id), - "meta.public": True, + **in_outbox(), + **by_remote_id(activity_url(item_id)), + **not_deleted(), + **is_public(), } ) if not doc: @@ -571,10 +570,10 @@ def outbox_activity_replies(item_id): _log_sig() data = DB.activities.find_one( { - "box": Box.OUTBOX.value, - "remote_id": activity_url(item_id), - "meta.deleted": False, - "meta.public": True, + **in_outbox(), + **by_remote_id(activity_url(item_id)), + **not_deleted(), + **is_public(), } ) if not data: @@ -584,9 +583,9 @@ def outbox_activity_replies(item_id): abort(404) q = { - "meta.deleted": False, - "meta.public": True, - "type": ActivityType.CREATE.value, + **is_public(), + **not_deleted(), + **by_type(ActivityType.CREATE), "activity.object.inReplyTo": obj.get_object().id, } diff --git a/core/meta.py b/core/meta.py index 40215dd..d00f1db 100644 --- a/core/meta.py +++ b/core/meta.py @@ -101,6 +101,10 @@ def follow_request_accepted() -> _SubQuery: return flag(MetaKey.FOLLOW_STATUS, FollowStatus.ACCEPTED.value) +def not_poll_answer() -> _SubQuery: + return flag(MetaKey.POLL_ANSWER, False) + + def not_in_reply_to() -> _SubQuery: return {"activity.object.inReplyTo": None} From 2f0e99bda0d870741cf482dedfade379dda279ac Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 3 Sep 2019 22:18:25 +0200 Subject: [PATCH 0680/1425] Tweak the README --- README.md | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index e4ab81b..8637ccb 100644 --- a/README.md +++ b/README.md @@ -18,20 +18,25 @@ ## Features - Implements a basic [ActivityPub](https://activitypub.rocks/) server (with federation) - - Compatible with [Mastodon](https://joinmastodon.org/) and others ([Pleroma](https://pleroma.social/), Plume, PixelFed, Hubzilla...) - - Also implements a remote follow compatible with Mastodon instances + - S2S (Server to Server) and C2S (Client to Server) protocols + - Compatible with [Mastodon](https://joinmastodon.org/) and others ([Pleroma](https://pleroma.social/), Misskey, Plume, PixelFed, Hubzilla...) - Exposes your outbox as a basic microblog - Support all content types from the Fediverse (`Note`, `Article`, `Page`, `Video`, `Image`, `Question`...) - - Comes with an admin UI with notifications and the stream of people you follow + - Comes with an admin UI with notifications and the stream of pOeople you follow + - Private "bookmark" support + - List support - Allows you to attach files to your notes - - Privacy-aware image upload endpoint that strip EXIF meta data before storing the file + - Cares about your privacy + - The image upload endpoint strips EXIF meta data before storing the file + - Every attachment/media is cached (or proxied) by the server - No JavaScript, **that's it**. Even the admin UI is pure HTML/CSS - (well except for the Emoji picker within the admin, but it's only few line of hand-written JavaScript) - Easy to customize (the theme is written Sass) - mobile-friendly theme - with dark and light version - - Microformats aware (exports `h-feed`, `h-entry`, `h-cards`, ...) - - Exports RSS/Atom/[JSON](https://jsonfeed.org/) feeds + - IndieWeb stuff + - Microformats aware (exports `h-feed`, `h-entry`, `h-cards`, ...) + - Exports RSS/Atom/[JSON](https://jsonfeed.org/) feeds - You stream/timeline is also available in an (authenticated) JSON feed - Comes with a tiny HTTP API to help posting new content and and read your inbox/notifications - Deployable with Docker (Docker compose for everything: dev, test and deployment) @@ -39,10 +44,9 @@ - U2F support - You can use your ActivityPub identity to login to other websites/app - Focused on testing - - Tested against the [official ActivityPub test suite](https://test.activitypub.rocks/) ([report submitted](https://github.com/w3c/activitypub/issues/308)) + - Tested against the [official ActivityPub test suite](https://test.activitypub.rocks/), see [the results](https://activitypub.rocks/implementation-report/) - [CI runs "federation" tests against two instances](https://d.a4.io/tsileo/microblog.pub) - Project is running 2 up-to-date instances ([here](https://microblog.pub) and [there](https://a4.io)) - - The core ActivityPub code/tests are in [Little Boxes](https://github.com/tsileo/little-boxes) (but needs some cleanup) - Manually tested against other major platforms @@ -57,7 +61,6 @@ Python is not needed on the host system. Note that all the generated data (config included) will be stored on the host (i.e. not only in Docker) in `config/` and `data/`. - ### Installation ```shell From 90609468f1c355519f39a200c3a82488e5ad6bf5 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 3 Sep 2019 22:24:30 +0200 Subject: [PATCH 0681/1425] Tweak reply processing --- core/activitypub.py | 12 ++++++++---- core/meta.py | 2 ++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/core/activitypub.py b/core/activitypub.py index 4522484..8bc64ec 100644 --- a/core/activitypub.py +++ b/core/activitypub.py @@ -763,13 +763,17 @@ def handle_replies(create: ap.Create) -> None: ) return None + in_reply_to_data = { + MetaKey.IN_REPLY_TO: in_reply_to, + MetaKey.IN_REPLY_TO_URL: reply.get_url(), + } # Update the activity to save some data about the reply if reply.get_actor().id == create.get_actor().id: - in_reply_to_data = {MetaKey.IN_REPLY_TO_SELF: True} + in_reply_to_data.update({MetaKey.IN_REPLY_TO_SELF: True}) else: - in_reply_to_data = { - MetaKey.IN_REPLY_TO_ACTOR: reply.get_actor().to_dict(embed=True) - } + in_reply_to_data.update( + {MetaKey.IN_REPLY_TO_ACTOR: reply.get_actor().to_dict(embed=True)} + ) update_one_activity(by_remote_id(create.id), upsert(in_reply_to_data)) # It's a regular reply, try to increment the reply counter diff --git a/core/meta.py b/core/meta.py index d00f1db..bc2f5e7 100644 --- a/core/meta.py +++ b/core/meta.py @@ -54,6 +54,8 @@ class MetaKey(Enum): THREAD_ROOT_PARENT = "thread_root_parent" + IN_REPLY_TO = "in_reply_to" + IN_REPLY_TO_URL = "in_reply_to_url" IN_REPLY_TO_SELF = "in_reply_to_self" IN_REPLY_TO_ACTOR = "in_reply_to_actor" From 2bbee2ee54d831e5b4e63a8cabb3611e24533cf3 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Tue, 3 Sep 2019 22:54:01 +0200 Subject: [PATCH 0682/1425] Tweak the admin UI --- sass/base_theme.scss | 21 +++++++++++++++++++++ templates/new.html | 12 ++++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/sass/base_theme.scss b/sass/base_theme.scss index 437095b..84e2b55 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -350,6 +350,27 @@ form.action-form { #admin { margin-top: 50px; } +.tabbar { + margin-bottom:50px; +} +.tab { + padding: 10px; + text-decoration: none; +} +.tab.selected { + background: $color-menu-background; + color: $primary-color; + border-top: 1px solid $primary-color; + border-right: 1px solid $primary-color; + border-left: 1px solid $primary-color; + padding: 9px; + +} +.tab:hover { + text-decoration: none; + background: $color-menu-background; + color: $color-light; +} textarea, input, select { background: $color-menu-background; padding: 10px; diff --git a/templates/new.html b/templates/new.html index fb5a26b..f422de1 100644 --- a/templates/new.html +++ b/templates/new.html @@ -12,11 +12,13 @@

    Replying to {{ content }}

    {{ utils.display_thread(thread) }} {% else %} +
    {% if request.args.get("question") == "1" %} -

    New question make it a note?

    +NoteQuestion {% else %} -

    New note make it a question?

    +NoteQuestion {% endif %} +
    {% endif %} @@ -37,13 +39,15 @@ {% endfor %}

    - + {% if request.args.get("question") == "1" %}

    Open for: - + @@ -163,6 +163,7 @@ new follower {% if item.meta.notification_unread %}new{% endif %} {{ (item.activity.published or item.meta.published) | format_timeago }} + profile {% if item.meta.notification_follows_back %}already following {% else %}

    @@ -182,9 +183,23 @@ you started following {% if item.meta.notification_unread %}new{% endif %} {{ (item.activity.published or item.meta.published) | format_timeago }} + profile {% if item.meta.notification_follows_back %}follows you back{% endif %}
    +
    + {{ utils.display_actor_inline(item.meta.actor, size=50) }} +
    + + {% elif item | has_type('Reject') %} +
    + rejected your follow request + {% if item.meta.notification_unread %}new{% endif %} + {{ (item.activity.published or item.meta.published) | format_timeago }} + profile + {% if item.meta.notification_follows_back %}follows you{% endif %} +
    +
    {{ utils.display_actor_inline(item.meta.actor, size=50) }}
    diff --git a/templates/utils.html b/templates/utils.html index 32683fa..26a392c 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -204,7 +204,8 @@ self reply {% endif %} {% for a in obj.attachment %} {% if (a.mediaType and a.mediaType.startswith("image/")) or (a.type and a.type == 'Image') %} - + + {% elif (a.mediaType and a.mediaType.startswith("video/")) %}
    • {% elif (a.mediaType and a.mediaType.startswith("audio/")) %} From 6a20f30bad8172c2a54d6e502dc72f394f80174c Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 6 Sep 2019 00:24:20 +0200 Subject: [PATCH 0691/1425] Fix follower handling --- app.py | 10 +++++++++- blueprints/admin.py | 2 ++ core/inbox.py | 15 +++++---------- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/app.py b/app.py index f51f0b1..2259102 100644 --- a/app.py +++ b/app.py @@ -60,6 +60,7 @@ from core.meta import by_object_id from core.meta import by_remote_id from core.meta import by_type from core.meta import by_visibility +from core.meta import follow_request_accepted from core.meta import in_inbox from core.meta import in_outbox from core.meta import is_public @@ -145,6 +146,7 @@ def inject_config(): following_q = { **in_outbox(), **by_type(ActivityType.FOLLOW), + **follow_request_accepted(), **not_undo(), **not_deleted(), } @@ -849,7 +851,13 @@ def followers(): @app.route("/following") def following(): - q = {**in_outbox(), **by_type(ActivityType.FOLLOW), **not_undo()} + q = { + **in_outbox(), + **by_type(ActivityType.FOLLOW), + **not_deleted(), + **follow_request_accepted(), + **not_undo(), + } if is_api_request(): _log_sig() diff --git a/blueprints/admin.py b/blueprints/admin.py index 5486896..6d51602 100644 --- a/blueprints/admin.py +++ b/blueprints/admin.py @@ -379,6 +379,7 @@ def admin_notifications() -> _Response: "activity.object": {"$regex": f"^{config.BASE_URL}"}, } followed_query = {"type": ap.ActivityType.ACCEPT.value} + rejected_query = {"type": ap.ActivityType.REJECT.value} q = { "box": Box.INBOX.value, "$or": [ @@ -387,6 +388,7 @@ def admin_notifications() -> _Response: replies_query, new_followers_query, followed_query, + rejected_query, unfollow_query, likes_query, ], diff --git a/core/inbox.py b/core/inbox.py index 939f2f0..93cace5 100644 --- a/core/inbox.py +++ b/core/inbox.py @@ -176,15 +176,10 @@ def _follow_process_inbox(activity: ap.Follow, new_meta: _NewMeta) -> None: post_to_outbox(accept) -def _update_follow_status(follow: ap.BaseActivity, status: FollowStatus) -> None: - if not follow.has_type(ap.Follow) or not is_from_outbox(follow): - _logger.warning( - "received an Accept/Reject from an unexpected activity: {follow!r}" - ) - return None - +def _update_follow_status(follow_id: str, status: FollowStatus) -> None: + _logger.info(f"{follow_id} is {status}") update_one_activity( - by_remote_id(follow.id), upsert({MetaKey.FOLLOW_STATUS: status.value}) + by_remote_id(follow_id), upsert({MetaKey.FOLLOW_STATUS: status.value}) ) @@ -192,14 +187,14 @@ def _update_follow_status(follow: ap.BaseActivity, status: FollowStatus) -> None def _accept_process_inbox(activity: ap.Accept, new_meta: _NewMeta) -> None: _logger.info(f"process_inbox activity={activity!r}") # Set a flag on the follow - follow = activity.get_object() + follow = activity.get_object_id() _update_follow_status(follow, FollowStatus.ACCEPTED) @process_inbox.register def _reject_process_inbox(activity: ap.Reject, new_meta: _NewMeta) -> None: _logger.info(f"process_inbox activity={activity!r}") - follow = activity.get_object() + follow = activity.get_object_id() _update_follow_status(follow, FollowStatus.REJECTED) From ced62409460666c50fc24691225f7d5c207e5c03 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 6 Sep 2019 00:25:06 +0200 Subject: [PATCH 0692/1425] Remove unused import --- core/inbox.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/inbox.py b/core/inbox.py index 93cace5..fc97ee9 100644 --- a/core/inbox.py +++ b/core/inbox.py @@ -9,7 +9,6 @@ from little_boxes.errors import NotAnActivityError import config from core.activitypub import _answer_key from core.activitypub import handle_replies -from core.activitypub import is_from_outbox from core.activitypub import post_to_outbox from core.activitypub import update_cached_actor from core.db import DB From 786a709a40fb707675d502d1a82f0f3a7854f5c5 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 6 Sep 2019 19:51:14 +0200 Subject: [PATCH 0693/1425] Support activities forwarded in a Read --- core/activitypub.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/activitypub.py b/core/activitypub.py index 30d48cb..044a8ef 100644 --- a/core/activitypub.py +++ b/core/activitypub.py @@ -219,6 +219,13 @@ def post_to_inbox(activity: ap.BaseActivity) -> None: Tasks.cache_actor(activity.id) return + # Honk forwards activities in a Read, process them as replies + if activity.has_type(ap.ActivityType.READ): + Tasks.process_reply(activity.get_object_id()) + return + + # TODO(tsileo): support ignore from Honk + # Hubzilla forwards activities in a Create, process them as possible replies if activity.has_type(ap.ActivityType.CREATE) and server(activity.id) != server( activity.get_object_id() From 119bbdcede2e077a13d35475a814505dbee20011 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 6 Sep 2019 20:04:23 +0200 Subject: [PATCH 0694/1425] Add migration for bugfix --- core/migrations.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/core/migrations.py b/core/migrations.py index db39438..606f0f7 100644 --- a/core/migrations.py +++ b/core/migrations.py @@ -318,3 +318,34 @@ class _20190901_MetaHashtagsAndMentions(Migration): except Exception: logger.exception(f"failed to process activity {data!r}") + + +class _20190906_RedoFollowFollowBack(_20190901_FollowFollowBackMigrationFix): + """Add the new meta flags for tracking accepted/rejected status and following/follows back info.""" + + +class _20190906_InReplyToMigration(Migration): + def migrate(self) -> None: + for data in find_activities( + {**by_type(ap.ActivityType.CREATE), **not_deleted()} + ): + try: + in_reply_to = data["activity"]["object"].get("inReplyTo") + if in_reply_to: + update_one_activity( + by_remote_id(data["remote_id"]), + upsert({MetaKey.IN_REPLY_TO: in_reply_to}), + ) + except Exception: + logger.exception(f"failed to process activity {data!r}") + + for data in DB.replies.find({**not_deleted()}): + try: + in_reply_to = data["activity"].get("inReplyTo") + if in_reply_to: + DB.replies.update_one( + by_remote_id(data["remote_id"]), + upsert({MetaKey.IN_REPLY_TO: in_reply_to}), + ) + except Exception: + logger.exception(f"failed to process activity {data!r}") From 606a2dfa684f880ba93989475afecc31fe78ea5d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 6 Sep 2019 20:43:49 +0200 Subject: [PATCH 0695/1425] Add support for profile metadata --- config.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/config.py b/config.py index 7a44fac..18ce40d 100644 --- a/config.py +++ b/config.py @@ -10,6 +10,7 @@ from itsdangerous import JSONWebSignatureSerializer from little_boxes import strtobool from little_boxes.activitypub import CTX_AS as AP_DEFAULT_CTX from pymongo import MongoClient +from bleach import linkify import sass from utils.emojis import _load_emojis @@ -128,6 +129,13 @@ def _admin_jwt_token() -> str: ADMIN_API_KEY = get_secret_key("admin_api_key", _admin_jwt_token) +attachments = [] +if conf.get("profile_metadata"): + for key, value in conf["profile_metadata"].items(): + attachments.append( + {"type": "PropertyValue", "name": key, "value": linkify(value)} + ) + ME = { "@context": DEFAULT_CTX, "type": "Person", @@ -143,7 +151,7 @@ ME = { "endpoints": {}, "url": ID, "manuallyApprovesFollowers": False, - "attachment": [], + "attachment": attachments, "icon": { "mediaType": mimetypes.guess_type(ICON_URL)[0], "type": "Image", From 31902ddb46828fc710e6b87a9d3295fe519d9e2a Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 6 Sep 2019 20:49:18 +0200 Subject: [PATCH 0696/1425] Remov silly message on the remote follow page --- templates/remote_follow.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/remote_follow.html b/templates/remote_follow.html index 413c0d1..7a4a454 100644 --- a/templates/remote_follow.html +++ b/templates/remote_follow.html @@ -5,12 +5,12 @@ {% block content %}
    {% include "header.html" %} -

    You're about to follow me \o/

    +

    Remote follow @{{ config.USERNAME }}@{{ config.DOMAIN }}

    - +
    From d035e521d3f0fb4f20e15cef996c7879f356ad0c Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 7 Sep 2019 09:47:58 +0200 Subject: [PATCH 0697/1425] Tweak media proxy --- app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 2259102..58895e3 100644 --- a/app.py +++ b/app.py @@ -271,7 +271,8 @@ def proxy(scheme: str, url: str) -> Any: req_headers = { k: v for k, v in dict(request.headers).items() - if k.lower() not in ["host", "cookie"] + if k.lower() not in ["host", "cookie", "", "x-forwarded-for", "x-real-ip"] + and not k.lower().startswith("broxy-") } req_headers["Host"] = urlparse(url).netloc resp = requests.get(url, stream=True, headers=req_headers) From ae809cc46d57206483ef8ac2c31ded10c834d0b9 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sat, 7 Sep 2019 10:35:28 +0200 Subject: [PATCH 0698/1425] Tweak note rendering --- templates/utils.html | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/templates/utils.html b/templates/utils.html index 26a392c..99a2ea4 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -246,7 +246,8 @@ self reply
    {% if perma %} -{{ obj.published | format_time }} + + {% endif %} {% if meta.count_reply and obj.id | is_from_outbox %}{{ meta.count_reply }} replies @@ -349,7 +350,9 @@ self reply {% if meta.object_visibility | visibility_is_public %} {% if obj | url_or_id | get_url | is_from_outbox %} -permalink +{% if not perma %} +permalink +{% endif %} {% else %} source {% if session.logged_in %} @@ -357,9 +360,7 @@ self reply {% endif %} {% endif %} {% endif %} -{% if session.logged_in %} {{ meta.object_visibility | visibility }} -{% endif %}
    From 30eb21203c1ef3f98d28b649f48269b36beea695 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 8 Sep 2019 09:51:30 +0200 Subject: [PATCH 0699/1425] Add support for CW, and tweak the new note form for mobile --- blueprints/api.py | 1 + templates/new.html | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/blueprints/api.py b/blueprints/api.py index e34a9bc..40bad69 100644 --- a/blueprints/api.py +++ b/blueprints/api.py @@ -432,6 +432,7 @@ def api_new_note() -> _Response: attributedTo=MY_PERSON.id, cc=list(set(cc)), to=list(set(to)), + summary=_user_api_arg("summary", default=""), content=content, tag=tags, source={"mediaType": "text/markdown", "content": source}, diff --git a/templates/new.html b/templates/new.html index f422de1..7db41b0 100644 --- a/templates/new.html +++ b/templates/new.html @@ -23,13 +23,15 @@
    - {% for v in visibility %} {% endfor %} {% if reply %}{% endif %} +

    +

    {% for emoji in emojis %} {{ emoji | emojify | safe }} @@ -39,7 +41,7 @@ {% endfor %}

    - + {% if request.args.get("question") == "1" %} From 623261b8327d93bcf4dd9b026bac0fde66f57d3e Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 8 Sep 2019 10:34:34 +0200 Subject: [PATCH 0700/1425] Tweak the template --- sass/base_theme.scss | 2 +- templates/header.html | 4 ++-- templates/new.html | 6 ++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/sass/base_theme.scss b/sass/base_theme.scss index 84e2b55..15ee2fd 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -89,7 +89,7 @@ a:hover { } } } -#header { +header { margin-bottom: 70px; .title { diff --git a/templates/header.html b/templates/header.html index 4c488e5..41d1d0a 100644 --- a/templates/header.html +++ b/templates/header.html @@ -1,5 +1,5 @@ {% if not request.path.startswith('/admin') %} - + {% endif %} diff --git a/templates/new.html b/templates/new.html index 7db41b0..8755170 100644 --- a/templates/new.html +++ b/templates/new.html @@ -42,7 +42,13 @@

    + +

    +

    +

    + +

    {% if request.args.get("question") == "1" %}
    From 51ed481bb51faa03cc94d0c30371ee18c1385f8b Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 8 Sep 2019 10:34:45 +0200 Subject: [PATCH 0701/1425] Add support for context --- blueprints/api.py | 32 +++++++++++++++++++++++++------- core/activitypub.py | 19 ++++++++++++++++++- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/blueprints/api.py b/blueprints/api.py index 40bad69..ae4447c 100644 --- a/blueprints/api.py +++ b/blueprints/api.py @@ -33,6 +33,7 @@ from config import _drop_db from core import feed from core.activitypub import activity_url from core.activitypub import post_to_outbox +from core.activitypub import new_context from core.meta import Box from core.meta import MetaKey from core.meta import _meta @@ -142,6 +143,7 @@ def api_delete() -> _Response: # Create the delete, same audience as the Create object delete = ap.Delete( + context=new_context(note), actor=ID, object=ap.Tombstone(id=note.id).to_dict(embed=True), to=note.to, @@ -169,6 +171,7 @@ def api_boost() -> _Response: to=[MY_PERSON.followers, note.attributedTo], cc=[ap.AS_PUBLIC], published=now(), + context=new_context(note), ) announce_id = post_to_outbox(announce) @@ -202,6 +205,7 @@ def api_vote() -> _Response: to=note.get_actor().id, name=choice, tag=[], + context=new_context(note), inReplyTo=note.id, ) raw_note["@context"] = config.DEFAULT_CTX @@ -232,7 +236,14 @@ def api_like() -> _Response: else: to = [note.get_actor().id] - like = ap.Like(object=note.id, actor=MY_PERSON.id, to=to, cc=cc, published=now()) + like = ap.Like( + object=note.id, + actor=MY_PERSON.id, + to=to, + cc=cc, + published=now(), + context=new_context(note), + ) like_id = post_to_outbox(like) @@ -301,6 +312,7 @@ def api_undo() -> _Response: undo = ap.Undo( actor=MY_PERSON.id, + context=new_context(obj), object=obj.to_dict(embed=True, embed_object_id_only=True), published=now(), to=obj.to, @@ -421,12 +433,11 @@ def api_new_note() -> _Response: else: cc.append(reply.attributedTo) + context = new_context(reply) + for tag in tags: if tag["type"] == "Mention": - if visibility == ap.Visibility.DIRECT: - to.append(tag["href"]) - else: - cc.append(tag["href"]) + to.append(tag["href"]) raw_note = dict( attributedTo=MY_PERSON.id, @@ -437,6 +448,7 @@ def api_new_note() -> _Response: tag=tags, source={"mediaType": "text/markdown", "content": source}, inReplyTo=reply.id if reply else None, + context=context, ) if "file" in request.files and request.files["file"].filename: @@ -450,7 +462,7 @@ def api_new_note() -> _Response: raw_note["attachment"] = [ { "mediaType": mtype, - "name": rfilename, + "name": _user_api_arg("file_description", default=rfilename), "type": "Document", "url": f"{BASE_URL}/uploads/{oid}/{rfilename}", } @@ -508,6 +520,7 @@ def api_new_question() -> _Response: attributedTo=MY_PERSON.id, cc=list(set(cc)), to=[ap.AS_PUBLIC], + context=new_context(), content=content, tag=tags, source={"mediaType": "text/markdown", "content": source}, @@ -563,7 +576,12 @@ def api_follow() -> _Response: return _user_api_response(activity=existing["activity"]["id"]) follow = ap.Follow( - actor=MY_PERSON.id, object=actor, to=[actor], cc=[ap.AS_PUBLIC], published=now() + actor=MY_PERSON.id, + object=actor, + to=[actor], + cc=[ap.AS_PUBLIC], + published=now(), + context=new_context(), ) follow_id = post_to_outbox(follow) diff --git a/core/activitypub.py b/core/activitypub.py index 044a8ef..db154ae 100644 --- a/core/activitypub.py +++ b/core/activitypub.py @@ -272,12 +272,29 @@ def save_reply(activity: ap.BaseActivity, meta: Dict[str, Any] = {}) -> None: ) +def new_context(parent: Optional[ap.BaseActivity] = None) -> str: + """`context` is here to group related activities, it's not meant to be resolved. + We're just following the convention.""" + # Copy the context from the parent if any + if parent and (parent.context or parent.conversation): + if parent.context: + if isinstance(parent.context, str): + return parent.context + elif isinstance(parent.context, dict) and parent.context.get("id"): + return parent.context["id"] + return parent.conversation + + # Generate a new context + ctx_id = binascii.hexlify(os.urandom(12)).decode("utf-8") + return urljoin(BASE_URL, f"/contexts/{ctx_id}") + + def post_to_outbox(activity: ap.BaseActivity) -> str: if activity.has_type(ap.CREATE_TYPES): activity = activity.build_create() # Assign create a random ID - obj_id = binascii.hexlify(os.urandom(8)).decode("utf-8") + obj_id = binascii.hexlify(os.urandom(12)).decode("utf-8") uri = activity_url(obj_id) activity._data["id"] = uri if activity.has_type(ap.ActivityType.CREATE): From ab01fed24a1159df27291a16cfcefddd17b65d5d Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Sun, 8 Sep 2019 10:56:46 +0200 Subject: [PATCH 0702/1425] Improve template markup --- sass/base_theme.scss | 2 +- templates/header.html | 2 +- templates/layout.html | 12 ++++++------ templates/utils.html | 18 +++++++++--------- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/sass/base_theme.scss b/sass/base_theme.scss index 15ee2fd..d3104ee 100644 --- a/sass/base_theme.scss +++ b/sass/base_theme.scss @@ -89,7 +89,7 @@ a:hover { } } } -header { +header#header { margin-bottom: 70px; .title { diff --git a/templates/header.html b/templates/header.html index 41d1d0a..cb1a926 100644 --- a/templates/header.html +++ b/templates/header.html @@ -1,5 +1,5 @@ {% if not request.path.startswith('/admin') %} -
    +