Bump version to 3.3.3

Browsers are phasing out X-XSS-Protection, but Safari and IE still support it.

.circleci/config.yml

@@ -160,8 +160,45 @@ jobs:
           name: Create database
           command: ./bin/rails db:create
       - run:
-          name: Run migrations
+          command: ./bin/rails db:migrate VERSION=20171010025614
+          name: Run migrations up to v2.0.0
+      - run:
+          command: ./bin/rails tests:migrations:populate_v2
+          name: Populate database with test data
+      - run:
           command: ./bin/rails db:migrate
+          name: Run all remaining migrations
+
+  test-two-step-migrations:
+    <<: *defaults
+    docker:
+      - image: circleci/ruby:2.7-buster-node
+        environment: *ruby_environment
+      - image: circleci/postgres:12.2
+        environment:
+          POSTGRES_USER: root
+          POSTGRES_HOST_AUTH_METHOD: trust
+      - image: circleci/redis:5-alpine
+    steps:
+      - *attach_workspace
+      - *install_system_dependencies
+      - run:
+          command: ./bin/rails db:create
+          name: Create database
+      - run:
+          command: ./bin/rails db:migrate VERSION=20171010025614
+          name: Run migrations up to v2.0.0
+      - run:
+          command: ./bin/rails tests:migrations:populate_v2
+          name: Populate database with test data
+      - run:
+          command: ./bin/rails db:migrate
+          name: Run all pre-deployment migrations
+          evironment:
+            SKIP_POST_DEPLOYMENT_MIGRATIONS: true
+      - run:
+          command: ./bin/rails db:migrate
+          name: Run all post-deployment remaining migrations
 
   test-ruby2.7:
     <<: *defaults
@@ -233,6 +270,9 @@ workflows:
       - test-migrations:
           requires:
             - install-ruby2.7
+      - test-two-step-migrations:
+          requires:
+            - install-ruby2.7
       - test-ruby2.7:
           requires:
             - install-ruby2.7

.env.production.sample

@@ -4,6 +4,12 @@
 # not demonstrate all available configuration options. Please look at
 # https://docs.joinmastodon.org/admin/config/ for the full documentation.
 
+# Note that this file accepts slightly different syntax depending on whether
+# you are using `docker-compose` or not. In particular, if you use
+# `docker-compose`, the value of each declared variable will be taken verbatim,
+# including surrounding quotes.
+# See: https://github.com/mastodon/mastodon/issues/16895
+
 # Federation
 # ----------
 # This identifies your server and cannot be changed safely later

.github/workflows/build-image.yml

@@ -0,0 +1,34 @@
+name: Build container image
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "main"
+    tags:
+      - "*"
+jobs:
+  build-image:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - uses: docker/metadata-action@v3
+        id: meta
+        with:
+          images: tootsuite/mastodon
+          flavor: |
+            latest=false
+          tags: |
+            type=edge,branch=main
+            type=semver,pattern={{ raw }}
+      - uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          cache-from: type=registry,ref=tootsuite/mastodon:latest
+          cache-to: type=inline

CHANGELOG.md

@@ -3,6 +3,89 @@ Changelog
 
 All notable changes to this project will be documented in this file.
 
+## [3.3.3] - 2022-03-30
+### Security
+
+- Fix being able to bypass e-mail restrictions ([Gargron](https://github.com/mastodon/mastodon/pull/17909))
+
+## [3.3.2] - 2022-02-03
+### Fixed
+- Fix `mastodon:webpush:generate_vapid_key` task requiring a functional environment ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17338))
+- Fix spurious errors when receiving an Add activity for a private post ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17425))
+
+### Security
+- Fix error-prone SQL queries ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/15828))
+- Fix not compacting incoming signed JSON-LD activities ([puckipedia](https://github.com/mastodon/mastodon/pull/17426), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/17428)) (CVE-2022-24307)
+- Fix insufficient sanitization of report comments ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17430))
+- Fix stop condition of a Common Table Expression ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17427))
+- Disable legacy XSS filtering ([Wonderfall](https://github.com/mastodon/mastodon/pull/17289))
+
+## [3.3.1] - 2022-01-31
+### Added
+- Add more advanced migration tests ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17393))
+- Add github workflow to build Docker images ([unasuke](https://github.com/mastodon/mastodon/pull/16973), [Gargron](https://github.com/mastodon/mastodon/pull/16980), [Gargron](https://github.com/mastodon/mastodon/pull/17000))
+
+### Fixed
+- Update some dependencies that were broken or unavailable
+- Fix some old migrations failing when skipping releases ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17394))
+- Fix migrations script failing in certain edge cases ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17398))
+- Fix media redownload worker retrying on unexpected response codes ([Gargron](https://github.com/tootsuite/mastodon/pull/16111))
+- Fix thread resolve worker retrying when status no longer exists ([Gargron](https://github.com/tootsuite/mastodon/pull/16109))
+- Fix n+1 queries when rendering statuses in REST API ([abcang](https://github.com/tootsuite/mastodon/pull/15641))
+- Fix remote reporters not receiving suspend/unsuspend activities ([Gargron](https://github.com/tootsuite/mastodon/pull/16050))
+- Fix understanding (not fully qualified) `as:Public` and `Public` ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15948))
+- Fix actor update not being distributed on profile picture deletion ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15461))
+- Fix processing of incoming Delete activities ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/16084))
+- Fix processing of incoming Block activities ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15546))
+- Fix processing of incoming Update activities of unknown accounts ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15514))
+- Fix URIs of repeat follow requests not being recorded ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15662))
+- Fix error on requests with no `Digest` header ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15782))
+- Fix activity object not requiring signature in secure mode ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15592))
+- Fix database serialization failure returning HTTP 500 ([Gargron](https://github.com/tootsuite/mastodon/pull/16101))
+- Fix media processing getting stuck on too much stdin/stderr ([Gargron](https://github.com/tootsuite/mastodon/pull/16136))
+- Fix `tootctl maintenance fix-duplicates` failures ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15923), [ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15515))
+- Fix blocking someone not clearing up list feeds ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/16205))
+- Fix edge case where follow limit interferes with accepting a follow ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/16098))
+- Fix reports of already suspended accounts being recorded ([Gargron](https://github.com/tootsuite/mastodon/pull/16047))
+- Fix sign-up restrictions based on IP addresses not being enforced ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15607))
+- Fix race conditions on account migration creation ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15597))
+- Fix not being able to change world filter expiration back to “Never” ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15858))
+- Fix error when muting users with `duration` in REST API ([Tak](https://github.com/tootsuite/mastodon/pull/15516))
+- Fix wrong URL to custom CSS when `CDN_HOST` is used ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15927))
+- Fix `tootctl accounts unfollow` ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15639))
+- Fix `tootctl emoji import` wasting time on MacOS shadow files ([cortices](https://github.com/tootsuite/mastodon/pull/15430))
+- Fix `tootctl emoji import` not treating shortcodes as case-insensitive ([angristan](https://github.com/tootsuite/mastodon/pull/15738))
+- Fix some issues with SAML account creation ([Gargron](https://github.com/tootsuite/mastodon/pull/15222), [kaiyou](https://github.com/tootsuite/mastodon/pull/15511
+- Fix instance actor not being automatically created if it wasn't seeded properly ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15693))))
+- Fix app name, website and redirect URIs not having a maximum length ([Gargron](https://github.com/tootsuite/mastodon/pull/16042))
+- Fix some ActivityPub identifiers in server actor outbox ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16343))
+- Fix custom CSS path setting cookies and being uncacheable due to it ([tribela](https://github.com/mastodon/mastodon/pull/16314))
+- Fix some redis locks auto-releasing too fast ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16276), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/16291))
+- Fix migration script not being able to run if it fails midway ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16312))
+- Fix PWA not being usable from alternate domains ([HolgerHuo](https://github.com/mastodon/mastodon/pull/16714))
+- Fix scheduling a status decreasing status count ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16791))
+- Fix followers synchronization mechanism not working when URI has empty path ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16744))
+- Fix serialization of counts in REST API when user hides their network ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16418))
+- Fix `tootctl self-destruct` not sending delete activities for recently-suspended accounts ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16688))
+- Fix `mastodon:setup` generated env-file syntax ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16896), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/16976))
+- Fix link previews being incorrectly generated from earlier links ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16885))
+- Fix wrong `to`/`cc` values for remote groups in ActivityPub ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16700))
+- Fix mentions with non-ascii TLDs not being processed ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16689))
+- Fix authentication failures halfway through a sign-in attempt ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16607), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/16792))
+- Fix suspended accounts statuses being merged back into timelines ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16628))
+- Fix crash when encountering invalid account fields ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16598))
+- Fix invalid blurhash handling for remote activities ([noellabo](https://github.com/mastodon/mastodon/pull/16583))
+- Fix newlines being added to accout notes when an account moves ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16415), [noellabo](https://github.com/mastodon/mastodon/pull/16576))
+- Fix logging out from one browser logging out all other sessions ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16943))
+- Fix confusing error when WebFinger request returns empty document ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16986))
+- Fix upload of remote media with OpenStack Swift sometimes failing ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16998))
+- Fix Docker build ([tribela](https://github.com/mastodon/mastodon/pull/17188))
+
+### Security
+- Fix user notes not having a length limit ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16942))
+- Fix revoking a specific session not working ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16943))
+- Fix filtering DMs from non-followed users ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17042))
+
 ## [3.3.0] - 2020-12-27
 ### Added
 

Dockerfile

@@ -71,8 +71,8 @@ RUN npm install -g yarn && \
 COPY Gemfile* package.json yarn.lock /opt/mastodon/
 
 RUN cd /opt/mastodon && \
-  bundle config set deployment 'true' && \
-  bundle config set without 'development test' && \
+  bundle config set --local deployment 'true' && \
+  bundle config set --local without 'development test' && \
 	bundle install -j$(nproc) && \
 	yarn install --pure-lockfile
 

Gemfile

@@ -21,8 +21,6 @@ gem 'aws-sdk-s3', '~> 1.85', require: false
 gem 'fog-core', '<= 2.1.0'
 gem 'fog-openstack', '~> 0.3', require: false
 gem 'paperclip', '~> 6.0'
-gem 'paperclip-av-transcoder', '~> 0.6'
-gem 'streamio-ffmpeg', '~> 3.0'
 gem 'blurhash', '~> 0.1'
 
 gem 'active_model_serializers', '~> 0.10'

Gemfile.lock

@@ -75,8 +75,6 @@ GEM
     ast (2.4.1)
     attr_encrypted (3.1.0)
       encryptor (~> 3.0.0)
-    av (0.9.0)
-      cocaine (~> 0.5.3)
     awrence (1.1.1)
     aws-eventstream (1.1.0)
     aws-partitions (1.397.0)
@@ -151,8 +149,6 @@ GEM
     cld3 (3.3.0)
       ffi (>= 1.1.0, < 1.12.0)
     climate_control (0.2.0)
-    cocaine (0.5.8)
-      climate_control (>= 0.0.3, < 1.0)
     coderay (1.1.3)
     color_diff (0.1)
     concurrent-ruby (1.1.7)
@@ -346,9 +342,11 @@ GEM
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2020.0512)
-    mimemagic (0.3.5)
+    mimemagic (0.3.10)
+      nokogiri (~> 1)
+      rake
     mini_mime (1.0.2)
-    mini_portile2 (2.4.0)
+    mini_portile2 (2.7.1)
     minitest (5.14.2)
     msgpack (1.3.3)
     multi_json (1.15.0)
@@ -358,9 +356,10 @@ GEM
       net-ssh (>= 2.6.5, < 7.0.0)
     net-ssh (6.1.0)
     nio4r (2.5.4)
-    nokogiri (1.10.10)
-      mini_portile2 (~> 2.4.0)
-    nokogumbo (2.0.2)
+    nokogiri (1.13.1)
+      mini_portile2 (~> 2.7.0)
+      racc (~> 1.4)
+    nokogumbo (2.0.5)
       nokogiri (~> 1.8, >= 1.8.4)
     nsa (0.2.7)
       activesupport (>= 4.2, < 6)
@@ -391,9 +390,6 @@ GEM
       mime-types
       mimemagic (~> 0.3.0)
       terrapin (~> 0.6.0)
-    paperclip-av-transcoder (0.6.4)
-      av (~> 0.9.0)
-      paperclip (>= 2.5.2)
     parallel (1.20.1)
     parallel_tests (3.4.0)
       parallel
@@ -432,6 +428,7 @@ GEM
     pundit (2.1.0)
       activesupport (>= 3.0.0)
     raabro (1.3.3)
+    racc (1.6.0)
     rack (2.2.3)
     rack-attack (6.3.1)
       rack (>= 1.0, < 3)
@@ -605,8 +602,6 @@ GEM
     stackprof (0.2.16)
     statsd-ruby (1.4.0)
     stoplight (2.2.1)
-    streamio-ffmpeg (3.0.2)
-      multi_json (~> 1.8)
     strong_migrations (0.7.2)
       activerecord (>= 5)
     temple (0.8.2)
@@ -751,7 +746,6 @@ DEPENDENCIES
   omniauth-saml (~> 1.10)
   ox (~> 2.13)
   paperclip (~> 6.0)
-  paperclip-av-transcoder (~> 0.6)
   parallel (~> 1.20)
   parallel_tests (~> 3.4)
   parslet
@@ -797,7 +791,6 @@ DEPENDENCIES
   sprockets-rails (~> 3.2)
   stackprof
   stoplight (~> 2.2.1)
-  streamio-ffmpeg (~> 3.0)
   strong_migrations (~> 0.7)
   thor (~> 1.0)
   tty-prompt (~> 0.22)

app/controllers/activitypub/followers_synchronizations_controller.rb

@@ -19,11 +19,11 @@ class ActivityPub::FollowersSynchronizationsController < ActivityPub::BaseContro
   private
 
   def uri_prefix
-    signed_request_account.uri[/http(s?):\/\/[^\/]+\//]
+    signed_request_account.uri[Account::URL_PREFIX_RE]
   end
 
   def set_items
-    @items = @account.followers.where(Account.arel_table[:uri].matches(uri_prefix + '%', false, true)).pluck(:uri)
+    @items = @account.followers.where(Account.arel_table[:uri].matches("#{Account.sanitize_sql_like(uri_prefix)}/%", false, true)).or(@account.followers.where(uri: uri_prefix)).pluck(:uri)
   end
 
   def collection_presenter

app/controllers/activitypub/inboxes_controller.rb

@@ -5,7 +5,7 @@ class ActivityPub::InboxesController < ActivityPub::BaseController
   include JsonLdHelper
   include AccountOwnedConcern
 
-  before_action :skip_unknown_actor_delete
+  before_action :skip_unknown_actor_activity
   before_action :require_signature!
   skip_before_action :authenticate_user!
 
@@ -18,13 +18,13 @@ class ActivityPub::InboxesController < ActivityPub::BaseController
 
   private
 
-  def skip_unknown_actor_delete
-    head 202 if unknown_deleted_account?
+  def skip_unknown_actor_activity
+    head 202 if unknown_affected_account?
   end
 
-  def unknown_deleted_account?
+  def unknown_affected_account?
     json = Oj.load(body, mode: :strict)
-    json.is_a?(Hash) && json['type'] == 'Delete' && json['actor'].present? && json['actor'] == value_or_id(json['object']) && !Account.where(uri: json['actor']).exists?
+    json.is_a?(Hash) && %w(Delete Update).include?(json['type']) && json['actor'].present? && json['actor'] == value_or_id(json['object']) && !Account.where(uri: json['actor']).exists?
   rescue Oj::ParseError
     false
   end

app/controllers/activitypub/outboxes_controller.rb

@@ -29,7 +29,7 @@ class ActivityPub::OutboxesController < ActivityPub::BaseController
       )
     else
       ActivityPub::CollectionPresenter.new(
-        id: account_outbox_url(@account),
+        id: outbox_url,
         type: :ordered,
         size: @account.statuses_count,
         first: outbox_url(page: true),
@@ -47,11 +47,11 @@ class ActivityPub::OutboxesController < ActivityPub::BaseController
   end
 
   def next_page
-    account_outbox_url(@account, page: true, max_id: @statuses.last.id) if @statuses.size == LIMIT
+    outbox_url(page: true, max_id: @statuses.last.id) if @statuses.size == LIMIT
   end
 
   def prev_page
-    account_outbox_url(@account, page: true, min_id: @statuses.first.id) unless @statuses.empty?
+    outbox_url(page: true, min_id: @statuses.first.id) unless @statuses.empty?
   end
 
   def set_statuses

app/controllers/api/v1/accounts_controller.rb

@@ -42,7 +42,7 @@ class Api::V1::AccountsController < Api::BaseController
   end
 
   def mute
-    MuteService.new.call(current_user.account, @account, notifications: truthy_param?(:notifications), duration: (params[:duration] || 0))
+    MuteService.new.call(current_user.account, @account, notifications: truthy_param?(:notifications), duration: (params[:duration]&.to_i || 0))
     render json: @account, serializer: REST::RelationshipSerializer, relationships: relationships
   end
 

app/controllers/application_controller.rb

@@ -20,17 +20,16 @@ class ApplicationController < ActionController::Base
   helper_method :use_seamless_external_login?
   helper_method :whitelist_mode?
 
-  rescue_from ActionController::RoutingError, with: :not_found
-  rescue_from ActionController::InvalidAuthenticityToken, with: :unprocessable_entity
-  rescue_from ActionController::UnknownFormat, with: :not_acceptable
-  rescue_from ActionController::ParameterMissing, with: :bad_request
-  rescue_from Paperclip::AdapterRegistry::NoHandlerError, with: :bad_request
-  rescue_from ActiveRecord::RecordNotFound, with: :not_found
+  rescue_from ActionController::ParameterMissing, Paperclip::AdapterRegistry::NoHandlerError, with: :bad_request
   rescue_from Mastodon::NotPermittedError, with: :forbidden
-  rescue_from HTTP::Error, OpenSSL::SSL::SSLError, with: :internal_server_error
-  rescue_from Mastodon::RaceConditionError, Seahorse::Client::NetworkingError, Stoplight::Error::RedLight, with: :service_unavailable
+  rescue_from ActionController::RoutingError, ActiveRecord::RecordNotFound, with: :not_found
+  rescue_from ActionController::UnknownFormat, with: :not_acceptable
+  rescue_from ActionController::InvalidAuthenticityToken, with: :unprocessable_entity
   rescue_from Mastodon::RateLimitExceededError, with: :too_many_requests
 
+  rescue_from HTTP::Error, OpenSSL::SSL::SSLError, with: :internal_server_error
+  rescue_from Mastodon::RaceConditionError, Seahorse::Client::NetworkingError, Stoplight::Error::RedLight, ActiveRecord::SerializationFailure, with: :service_unavailable
+
   before_action :store_current_location, except: :raise_not_found, unless: :devise_controller?
   before_action :require_functional!, if: :user_signed_in?
 

app/controllers/auth/passwords_controller.rb

@@ -10,7 +10,6 @@ class Auth::PasswordsController < Devise::PasswordsController
     super do |resource|
       if resource.errors.empty?
         resource.session_activations.destroy_all
-        resource.forget_me!
       end
     end
   end

app/controllers/auth/registrations_controller.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 class Auth::RegistrationsController < Devise::RegistrationsController
-  include Devise::Controllers::Rememberable
   include RegistrationSpamConcern
 
   layout :determine_layout
@@ -30,8 +29,6 @@ class Auth::RegistrationsController < Devise::RegistrationsController
     super do |resource|
       if resource.saved_change_to_encrypted_password?
         resource.clear_other_sessions(current_session.session_id)
-        resource.forget_me!
-        remember_me(resource)
       end
     end
   end

app/controllers/auth/sessions_controller.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 class Auth::SessionsController < Devise::SessionsController
-  include Devise::Controllers::Rememberable
-
   layout 'auth'
 
   skip_before_action :require_no_authentication, only: [:create]
@@ -26,7 +24,6 @@ class Auth::SessionsController < Devise::SessionsController
   def create
     super do |resource|
       resource.update_sign_in!(request, new_sign_in: true)
-      remember_me(resource)
       flash.delete(:notice)
     end
   end
@@ -40,7 +37,7 @@ class Auth::SessionsController < Devise::SessionsController
   end
 
   def webauthn_options
-    user = find_user
+    user = User.find_by(id: session[:attempt_user_id])
 
     if user.webauthn_enabled?
       options_for_get = WebAuthn::Credential.options_for_get(
@@ -58,16 +55,20 @@ class Auth::SessionsController < Devise::SessionsController
   protected
 
   def find_user
-    if session[:attempt_user_id]
+    if user_params[:email].present?
+      find_user_from_params
+    elsif session[:attempt_user_id]
       User.find_by(id: session[:attempt_user_id])
-    else
-      user   = User.authenticate_with_ldap(user_params) if Devise.ldap_authentication
-      user ||= User.authenticate_with_pam(user_params) if Devise.pam_authentication
-      user ||= User.find_for_authentication(email: user_params[:email])
-      user
     end
   end
 
+  def find_user_from_params
+    user   = User.authenticate_with_ldap(user_params) if Devise.ldap_authentication
+    user ||= User.authenticate_with_pam(user_params) if Devise.pam_authentication
+    user ||= User.find_for_authentication(email: user_params[:email])
+    user
+  end
+
   def user_params
     params.require(:user).permit(:email, :password, :otp_attempt, :sign_in_token_attempt, credential: {})
   end

app/controllers/concerns/sign_in_token_authentication_concern.rb

@@ -16,21 +16,24 @@ module SignInTokenAuthenticationConcern
   end
 
   def authenticate_with_sign_in_token
-    user = self.resource = find_user
+    if user_params[:email].present?
+      user = self.resource = find_user_from_params
+      prompt_for_sign_in_token(user) if user&.external_or_valid_password?(user_params[:password])
+    elsif session[:attempt_user_id]
+      user = self.resource = User.find_by(id: session[:attempt_user_id])
+      return if user.nil?
 
-    if user.present? && session[:attempt_user_id].present? && session[:attempt_user_updated_at] != user.updated_at.to_s
-      restart_session
-    elsif user_params.key?(:sign_in_token_attempt) && session[:attempt_user_id]
-      authenticate_with_sign_in_token_attempt(user)
-    elsif user.present? && user.external_or_valid_password?(user_params[:password])
-      prompt_for_sign_in_token(user)
+      if session[:attempt_user_updated_at] != user.updated_at.to_s
+        restart_session
+      elsif user_params.key?(:sign_in_token_attempt)
+        authenticate_with_sign_in_token_attempt(user)
+      end
     end
   end
 
   def authenticate_with_sign_in_token_attempt(user)
     if valid_sign_in_token_attempt?(user)
       clear_attempt_from_session
-      remember_me(user)
       sign_in(user)
     else
       flash.now[:alert] = I18n.t('users.invalid_sign_in_token')

app/controllers/concerns/signature_verification.rb

@@ -133,6 +133,7 @@ module SignatureVerification
 
   def verify_body_digest!
     return unless signed_headers.include?('digest')
+    raise SignatureVerificationError, 'Digest header missing' unless request.headers.key?('Digest')
 
     digests = request.headers['Digest'].split(',').map { |digest| digest.split('=', 2) }.map { |key, value| [key.downcase, value] }
     sha256  = digests.assoc('sha-256')

app/controllers/concerns/two_factor_authentication_concern.rb

@@ -35,16 +35,20 @@ module TwoFactorAuthenticationConcern
   end
 
   def authenticate_with_two_factor
-    user = self.resource = find_user
+    if user_params[:email].present?
+      user = self.resource = find_user_from_params
+      prompt_for_two_factor(user) if user&.external_or_valid_password?(user_params[:password])
+    elsif session[:attempt_user_id]
+      user = self.resource = User.find_by(id: session[:attempt_user_id])
+      return if user.nil?
 
-    if user.present? && session[:attempt_user_id].present? && session[:attempt_user_updated_at] != user.updated_at.to_s
-      restart_session
-    elsif user.webauthn_enabled? && user_params.key?(:credential) && session[:attempt_user_id]
-      authenticate_with_two_factor_via_webauthn(user)
-    elsif user_params.key?(:otp_attempt) && session[:attempt_user_id]
-      authenticate_with_two_factor_via_otp(user)
-    elsif user.present? && user.external_or_valid_password?(user_params[:password])
-      prompt_for_two_factor(user)
+      if session[:attempt_user_updated_at] != user.updated_at.to_s
+        restart_session
+      elsif user.webauthn_enabled? && user_params.key?(:credential)
+        authenticate_with_two_factor_via_webauthn(user)
+      elsif user_params.key?(:otp_attempt)
+        authenticate_with_two_factor_via_otp(user)
+      end
     end
   end
 
@@ -53,7 +57,6 @@ module TwoFactorAuthenticationConcern
 
     if valid_webauthn_credential?(user, webauthn_credential)
       clear_attempt_from_session
-      remember_me(user)
       sign_in(user)
       render json: { redirect_path: root_path }, status: :ok
     else
@@ -64,7 +67,6 @@ module TwoFactorAuthenticationConcern
   def authenticate_with_two_factor_via_otp(user)
     if valid_otp_attempt?(user)
       clear_attempt_from_session
-      remember_me(user)
       sign_in(user)
     else
       flash.now[:alert] = I18n.t('users.invalid_otp_token')

app/controllers/custom_css_controller.rb

@@ -3,11 +3,16 @@
 class CustomCssController < ApplicationController
   skip_before_action :store_current_location
   skip_before_action :require_functional!
+  skip_before_action :update_user_sign_in
+  skip_before_action :set_session_activity
+
+  skip_around_action :set_locale
 
   before_action :set_cache_headers
 
   def show
     expires_in 3.minutes, public: true
+    request.session_options[:skip] = true
     render plain: Setting.custom_css || '', content_type: 'text/css'
   end
 end

app/controllers/follower_accounts_controller.rb

@@ -85,7 +85,7 @@ class FollowerAccountsController < ApplicationController
     if page_requested? || !@account.user_hides_network?
       # Return all fields
     else
-      %i(id type totalItems)
+      %i(id type total_items)
     end
   end
 end

app/controllers/following_accounts_controller.rb

@@ -85,7 +85,7 @@ class FollowingAccountsController < ApplicationController
     if page_requested? || !@account.user_hides_network?
       # Return all fields
     else
-      %i(id type totalItems)
+      %i(id type total_items)
     end
   end
 end

app/controllers/instance_actors_controller.rb

@@ -13,7 +13,7 @@ class InstanceActorsController < ApplicationController
   private
 
   def set_account
-    @account = Account.find(-99)
+    @account = Account.representative
   end
 
   def restrict_fields_to

app/controllers/media_proxy_controller.rb

@@ -45,7 +45,7 @@ class MediaProxyController < ApplicationController
   end
 
   def lock_options
-    { redis: Redis.current, key: "media_download:#{params[:id]}" }
+    { redis: Redis.current, key: "media_download:#{params[:id]}", autorelease: 15.minutes.seconds }
   end
 
   def reject_media?

app/controllers/settings/pictures_controller.rb

@@ -7,8 +7,12 @@ module Settings
 
     def destroy
       if valid_picture?
-        msg = I18n.t('generic.changes_saved_msg') if UpdateAccountService.new.call(@account, { @picture => nil, "#{@picture}_remote_url" => '' })
-        redirect_to settings_profile_path, notice: msg, status: 303
+        if UpdateAccountService.new.call(@account, { @picture => nil, "#{@picture}_remote_url" => '' })
+          ActivityPub::UpdateDistributionWorker.perform_async(@account.id)
+          redirect_to settings_profile_path, notice: I18n.t('generic.changes_saved_msg'), status: 303
+        else
+          redirect_to settings_profile_path
+        end
       else
         bad_request
       end

app/controllers/statuses_controller.rb

@@ -8,7 +8,7 @@ class StatusesController < ApplicationController
 
   layout 'public'
 
-  before_action :require_signature!, only: :show, if: -> { request.format == :json && authorized_fetch_mode? }
+  before_action :require_signature!, only: [:show, :activity], if: -> { request.format == :json && authorized_fetch_mode? }
   before_action :set_status
   before_action :set_instance_presenter
   before_action :set_link_headers

app/helpers/context_helper.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module ContextHelper
+  NAMED_CONTEXT_MAP = {
+    activitystreams: 'https://www.w3.org/ns/activitystreams',
+    security: 'https://w3id.org/security/v1',
+  }.freeze
+
+  CONTEXT_EXTENSION_MAP = {
+    manually_approves_followers: { 'manuallyApprovesFollowers' => 'as:manuallyApprovesFollowers' },
+    sensitive: { 'sensitive' => 'as:sensitive' },
+    hashtag: { 'Hashtag' => 'as:Hashtag' },
+    moved_to: { 'movedTo' => { '@id' => 'as:movedTo', '@type' => '@id' } },
+    also_known_as: { 'alsoKnownAs' => { '@id' => 'as:alsoKnownAs', '@type' => '@id' } },
+    emoji: { 'toot' => 'http://joinmastodon.org/ns#', 'Emoji' => 'toot:Emoji' },
+    featured: { 'toot' => 'http://joinmastodon.org/ns#', 'featured' => { '@id' => 'toot:featured', '@type' => '@id' }, 'featuredTags' => { '@id' => 'toot:featuredTags', '@type' => '@id' } },
+    property_value: { 'schema' => 'http://schema.org#', 'PropertyValue' => 'schema:PropertyValue', 'value' => 'schema:value' },
+    atom_uri: { 'ostatus' => 'http://ostatus.org#', 'atomUri' => 'ostatus:atomUri' },
+    conversation: { 'ostatus' => 'http://ostatus.org#', 'inReplyToAtomUri' => 'ostatus:inReplyToAtomUri', 'conversation' => 'ostatus:conversation' },
+    focal_point: { 'toot' => 'http://joinmastodon.org/ns#', 'focalPoint' => { '@container' => '@list', '@id' => 'toot:focalPoint' } },
+    identity_proof: { 'toot' => 'http://joinmastodon.org/ns#', 'IdentityProof' => 'toot:IdentityProof' },
+    blurhash: { 'toot' => 'http://joinmastodon.org/ns#', 'blurhash' => 'toot:blurhash' },
+    discoverable: { 'toot' => 'http://joinmastodon.org/ns#', 'discoverable' => 'toot:discoverable' },
+    voters_count: { 'toot' => 'http://joinmastodon.org/ns#', 'votersCount' => 'toot:votersCount' },
+    olm: { 'toot' => 'http://joinmastodon.org/ns#', 'Device' => 'toot:Device', 'Ed25519Signature' => 'toot:Ed25519Signature', 'Ed25519Key' => 'toot:Ed25519Key', 'Curve25519Key' => 'toot:Curve25519Key', 'EncryptedMessage' => 'toot:EncryptedMessage', 'publicKeyBase64' => 'toot:publicKeyBase64', 'deviceId' => 'toot:deviceId', 'claim' => { '@type' => '@id', '@id' => 'toot:claim' }, 'fingerprintKey' => { '@type' => '@id', '@id' => 'toot:fingerprintKey' }, 'identityKey' => { '@type' => '@id', '@id' => 'toot:identityKey' }, 'devices' => { '@type' => '@id', '@id' => 'toot:devices' }, 'messageFranking' => 'toot:messageFranking', 'messageType' => 'toot:messageType', 'cipherText' => 'toot:cipherText' },
+    suspended: { 'toot' => 'http://joinmastodon.org/ns#', 'suspended' => 'toot:suspended' },
+  }.freeze
+
+  def full_context
+    serialized_context(NAMED_CONTEXT_MAP, CONTEXT_EXTENSION_MAP)
+  end
+
+  def serialized_context(named_contexts_map, context_extensions_map)
+    context_array = []
+
+    named_contexts     = named_contexts_map.keys
+    context_extensions = context_extensions_map.keys
+
+    named_contexts.each do |key|
+      context_array << NAMED_CONTEXT_MAP[key]
+    end
+
+    extensions = context_extensions.each_with_object({}) do |key, h|
+      h.merge!(CONTEXT_EXTENSION_MAP[key])
+    end
+
+    context_array << extensions unless extensions.empty?
+
+    if context_array.size == 1
+      context_array.first
+    else
+      context_array
+    end
+  end
+end

app/helpers/jsonld_helper.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 module JsonLdHelper
+  include ContextHelper
+
   def equals_or_includes?(haystack, needle)
     haystack.is_a?(Array) ? haystack.include?(needle) : haystack == needle
   end
@@ -63,6 +65,84 @@ module JsonLdHelper
     graph.dump(:normalize)
   end
 
+  def compact(json)
+    compacted = JSON::LD::API.compact(json.without('signature'), full_context, documentLoader: method(:load_jsonld_context))
+    compacted['signature'] = json['signature']
+    compacted
+  end
+
+  # Patches a JSON-LD document to avoid compatibility issues on redistribution
+  #
+  # Since compacting a JSON-LD document against Mastodon's built-in vocabulary
+  # means other extension namespaces will be expanded, malformed JSON-LD
+  # attributes lost, and some values “unexpectedly” compacted this method
+  # patches the following likely sources of incompatibility:
+  # - 'https://www.w3.org/ns/activitystreams#Public' being compacted to
+  #   'as:Public' (for instance, pre-3.4.0 Mastodon does not understand
+  #   'as:Public')
+  # - single-item arrays being compacted to the item itself (`[foo]` being
+  #   compacted to `foo`)
+  #
+  # It is not always possible for `patch_for_forwarding!` to produce a document
+  # deemed safe for forwarding. Use `safe_for_forwarding?` to check the status
+  # of the output document.
+  #
+  # @param original [Hash] The original JSON-LD document used as reference
+  # @param compacted [Hash] The compacted JSON-LD document to be patched
+  # @return [void]
+  def patch_for_forwarding!(original, compacted)
+    original.without('@context', 'signature').each do |key, value|
+      next if value.nil? || !compacted.key?(key)
+
+      compacted_value = compacted[key]
+      if value.is_a?(Hash) && compacted_value.is_a?(Hash)
+        patch_for_forwarding!(value, compacted_value)
+      elsif value.is_a?(Array)
+        compacted_value = [compacted_value] unless compacted_value.is_a?(Array)
+        return if value.size != compacted_value.size
+
+        compacted[key] = value.zip(compacted_value).map do |v, vc|
+          if v.is_a?(Hash) && vc.is_a?(Hash)
+            patch_for_forwarding!(v, vc)
+            vc
+          elsif v == 'https://www.w3.org/ns/activitystreams#Public' && vc == 'as:Public'
+            v
+          else
+            vc
+          end
+        end
+      elsif value == 'https://www.w3.org/ns/activitystreams#Public' && compacted_value == 'as:Public'
+        compacted[key] = value
+      end
+    end
+  end
+
+  # Tests whether a JSON-LD compaction is deemed safe for redistribution,
+  # that is, if it doesn't change its meaning to consumers that do not actually
+  # handle JSON-LD, but rely on values being serialized in a certain way.
+  #
+  # See `patch_for_forwarding!` for details.
+  #
+  # @param original [Hash] The original JSON-LD document used as reference
+  # @param compacted [Hash] The compacted JSON-LD document to be patched
+  # @return [Boolean] Whether the patched document is deemed safe
+  def safe_for_forwarding?(original, compacted)
+    original.without('@context', 'signature').all? do |key, value|
+      compacted_value = compacted[key]
+      return false unless value.class == compacted_value.class
+
+      if value.is_a?(Hash)
+        safe_for_forwarding?(value, compacted_value)
+      elsif value.is_a?(Array)
+        value.zip(compacted_value).all? do |v, vc|
+          v.is_a?(Hash) ? (vc.is_a?(Hash) && safe_for_forwarding?(v, vc)) : v == vc
+        end
+      else
+        value == compacted_value
+      end
+    end
+  end
+
   def fetch_resource(uri, id, on_behalf_of = nil)
     unless id
       json = fetch_resource_without_id_validation(uri, on_behalf_of)

app/lib/account_reach_finder.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class AccountReachFinder
+  def initialize(account)
+    @account = account
+  end
+
+  def inboxes
+    (followers_inboxes + reporters_inboxes + relay_inboxes).uniq
+  end
+
+  private
+
+  def followers_inboxes
+    @account.followers.inboxes
+  end
+
+  def reporters_inboxes
+    Account.where(id: @account.targeted_reports.select(:account_id)).inboxes
+  end
+
+  def relay_inboxes
+    Relay.enabled.pluck(:inbox_url)
+  end
+end

app/lib/activitypub/activity.rb

@@ -144,7 +144,7 @@ class ActivityPub::Activity
   end
 
   def delete_later!(uri)
-    redis.setex("delete_upon_arrival:#{@account.id}:#{uri}", 6.hours.seconds, uri)
+    redis.setex("delete_upon_arrival:#{@account.id}:#{uri}", 6.hours.seconds, true)
   end
 
   def status_from_object
@@ -210,12 +210,22 @@ class ActivityPub::Activity
     end
   end
 
-  def lock_or_return(key, expire_after = 7.days.seconds)
+  def lock_or_return(key, expire_after = 2.hours.seconds)
     yield if redis.set(key, true, nx: true, ex: expire_after)
   ensure
     redis.del(key)
   end
 
+  def lock_or_fail(key, expire_after = 15.minutes.seconds)
+    RedisLock.acquire({ redis: Redis.current, key: key, autorelease: expire_after }) do |lock|
+      if lock.acquired?
+        yield
+      else
+        raise Mastodon::RaceConditionError
+      end
+    end
+  end
+
   def fetch?
     !@options[:delivery]
   end

app/lib/activitypub/activity/add.rb

@@ -7,7 +7,7 @@ class ActivityPub::Activity::Add < ActivityPub::Activity
     status   = status_from_uri(object_uri)
     status ||= fetch_remote_original_status
 
-    return unless !status.nil? && status.account_id == @account.id && !@account.pinned?(status)
+    return unless !status.nil? && status.account_id == @account.id && !@account.pinned?(status) && status.distributable?
 
     StatusPin.create!(account: @account, status: status)
   end

app/lib/activitypub/activity/announce.rb

@@ -4,29 +4,25 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
   def perform
     return reject_payload! if delete_arrived_first?(@json['id']) || !related_to_local_activity?
 
-    RedisLock.acquire(lock_options) do |lock|
-      if lock.acquired?
-        original_status = status_from_object
+    lock_or_fail("announce:#{@object['id']}") do
+      original_status = status_from_object
 
-        return reject_payload! if original_status.nil? || !announceable?(original_status)
+      return reject_payload! if original_status.nil? || !announceable?(original_status)
 
-        @status = Status.find_by(account: @account, reblog: original_status)
+      @status = Status.find_by(account: @account, reblog: original_status)
 
-        return @status unless @status.nil?
+      return @status unless @status.nil?
 
-        @status = Status.create!(
-          account: @account,
-          reblog: original_status,
-          uri: @json['id'],
-          created_at: @json['published'],
-          override_timestamps: @options[:override_timestamps],
-          visibility: visibility_from_audience
-        )
+      @status = Status.create!(
+        account: @account,
+        reblog: original_status,
+        uri: @json['id'],
+        created_at: @json['published'],
+        override_timestamps: @options[:override_timestamps],
+        visibility: visibility_from_audience
+      )
 
-        distribute(@status)
-      else
-        raise Mastodon::RaceConditionError
-      end
+      distribute(@status)
     end
 
     @status
@@ -43,9 +39,9 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
   end
 
   def visibility_from_audience
-    if audience_to.include?(ActivityPub::TagManager::COLLECTIONS[:public])
+    if audience_to.any? { |to| ActivityPub::TagManager.instance.public_collection?(to) }
       :public
-    elsif audience_cc.include?(ActivityPub::TagManager::COLLECTIONS[:public])
+    elsif audience_cc.any? { |cc| ActivityPub::TagManager.instance.public_collection?(cc) }
       :unlisted
     elsif audience_to.include?(@account.followers_url)
       :private
@@ -69,8 +65,4 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
   def reblog_of_local_status?
     status_from_uri(object_uri)&.account&.local?
   end
-
-  def lock_options
-    { redis: Redis.current, key: "announce:#{@object['id']}" }
-  end
 end

app/lib/activitypub/activity/block.rb

@@ -11,8 +11,13 @@ class ActivityPub::Activity::Block < ActivityPub::Activity
       return
     end
 
+    UnfollowService.new.call(@account, target_account) if @account.following?(target_account)
     UnfollowService.new.call(target_account, @account) if target_account.following?(@account)
+    RejectFollowService.new.call(target_account, @account) if target_account.requested?(@account)
 
-    @account.block!(target_account, uri: @json['id']) unless delete_arrived_first?(@json['id'])
+    unless delete_arrived_first?(@json['id'])
+      BlockWorker.perform_async(@account.id, target_account.id)
+      @account.block!(target_account, uri: @json['id'])
+    end
   end
 end

app/lib/activitypub/activity/create.rb

@@ -45,19 +45,15 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def create_status
     return reject_payload! if unsupported_object_type? || invalid_origin?(object_uri) || tombstone_exists? || !related_to_local_activity?
 
-    RedisLock.acquire(lock_options) do |lock|
-      if lock.acquired?
-        return if delete_arrived_first?(object_uri) || poll_vote? # rubocop:disable Lint/NonLocalExitFromIterator
+    lock_or_fail("create:#{object_uri}") do
+      return if delete_arrived_first?(object_uri) || poll_vote? # rubocop:disable Lint/NonLocalExitFromIterator
 
-        @status = find_existing_status
+      @status = find_existing_status
 
-        if @status.nil?
-          process_status
-        elsif @options[:delivered_to_account_id].present?
-          postprocess_audience_and_deliver
-        end
-      else
-        raise Mastodon::RaceConditionError
+      if @status.nil?
+        process_status
+      elsif @options[:delivered_to_account_id].present?
+        postprocess_audience_and_deliver
       end
     end
 
@@ -123,7 +119,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
   def process_audience
     (audience_to + audience_cc).uniq.each do |audience|
-      next if audience == ActivityPub::TagManager::COLLECTIONS[:public]
+      next if ActivityPub::TagManager.instance.public_collection?(audience)
 
       # Unlike with tags, there is no point in resolving accounts we don't already
       # know here, because silent mentions would only be used for local access
@@ -314,13 +310,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     poll = replied_to_status.preloadable_poll
     already_voted = true
 
-    RedisLock.acquire(poll_lock_options) do |lock|
-      if lock.acquired?
-        already_voted = poll.votes.where(account: @account).exists?
-        poll.votes.create!(account: @account, choice: poll.options.index(@object['name']), uri: object_uri)
-      else
-        raise Mastodon::RaceConditionError
-      end
+    lock_or_fail("vote:#{replied_to_status.poll_id}:#{@account.id}") do
+      already_voted = poll.votes.where(account: @account).exists?
+      poll.votes.create!(account: @account, choice: poll.options.index(@object['name']), uri: object_uri)
     end
 
     increment_voters_count! unless already_voted
@@ -356,9 +348,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def visibility_from_audience
-    if audience_to.include?(ActivityPub::TagManager::COLLECTIONS[:public])
+    if audience_to.any? { |to| ActivityPub::TagManager.instance.public_collection?(to) }
       :public
-    elsif audience_cc.include?(ActivityPub::TagManager::COLLECTIONS[:public])
+    elsif audience_cc.any? { |cc| ActivityPub::TagManager.instance.public_collection?(cc) }
       :unlisted
     elsif audience_to.include?(@account.followers_url)
       :private
@@ -455,10 +447,14 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def supported_blurhash?(blurhash)
-    components = blurhash.blank? ? nil : Blurhash.components(blurhash)
+    components = blurhash.blank? || !blurhash_valid_chars?(blurhash) ? nil : Blurhash.components(blurhash)
     components.present? && components.none? { |comp| comp > 5 }
   end
 
+  def blurhash_valid_chars?(blurhash)
+    /^[\w#$%*+-.:;=?@\[\]^{|}~]+$/.match?(blurhash)
+  end
+
   def skip_download?
     return @skip_download if defined?(@skip_download)
 
@@ -513,12 +509,4 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     poll.reload
     retry
   end
-
-  def lock_options
-    { redis: Redis.current, key: "create:#{object_uri}" }
-  end
-
-  def poll_lock_options
-    { redis: Redis.current, key: "vote:#{replied_to_status.poll_id}:#{@account.id}" }
-  end
 end

app/lib/activitypub/activity/delete.rb

@@ -20,33 +20,35 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
   def delete_note
     return if object_uri.nil?
 
-    unless invalid_origin?(object_uri)
-      RedisLock.acquire(lock_options) { |_lock| delete_later!(object_uri) }
-      Tombstone.find_or_create_by(uri: object_uri, account: @account)
+    lock_or_return("delete_status_in_progress:#{object_uri}", 5.minutes.seconds) do
+      unless invalid_origin?(object_uri)
+        # This lock ensures a concurrent `ActivityPub::Activity::Create` either
+        # does not create a status at all, or has finished saving it to the
+        # database before we try to load it.
+        # Without the lock, `delete_later!` could be called after `delete_arrived_first?`
+        # and `Status.find` before `Status.create!`
+        lock_or_fail("create:#{object_uri}") { delete_later!(object_uri) }
+
+        Tombstone.find_or_create_by(uri: object_uri, account: @account)
+      end
+
+      @status   = Status.find_by(uri: object_uri, account: @account)
+      @status ||= Status.find_by(uri: @object['atomUri'], account: @account) if @object.is_a?(Hash) && @object['atomUri'].present?
+
+      return if @status.nil?
+
+      forward! if @json['signature'].present? && @status.distributable?
+      delete_now!
     end
-
-    @status   = Status.find_by(uri: object_uri, account: @account)
-    @status ||= Status.find_by(uri: @object['atomUri'], account: @account) if @object.is_a?(Hash) && @object['atomUri'].present?
-
-    return if @status.nil?
-
-    if @status.distributable?
-      forward_for_reply
-      forward_for_reblogs
-    end
-
-    delete_now!
   end
 
-  def forward_for_reblogs
-    return if @json['signature'].blank?
+  def rebloggers_ids
+    return @rebloggers_ids if defined?(@rebloggers_ids)
+    @rebloggers_ids = @status.reblogs.includes(:account).references(:account).merge(Account.local).pluck(:account_id)
+  end
 
-    rebloggers_ids = @status.reblogs.includes(:account).references(:account).merge(Account.local).pluck(:account_id)
-    inboxes        = Account.where(id: ::Follow.where(target_account_id: rebloggers_ids).select(:account_id)).inboxes - [@account.preferred_inbox_url]
-
-    ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes) do |inbox_url|
-      [payload, rebloggers_ids.first, inbox_url]
-    end
+  def inboxes_for_reblogs
+    Account.where(id: ::Follow.where(target_account_id: rebloggers_ids).select(:account_id)).inboxes
   end
 
   def replied_to_status
@@ -58,13 +60,19 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
     !replied_to_status.nil? && replied_to_status.account.local?
   end
 
-  def forward_for_reply
-    return unless @json['signature'].present? && reply_to_local?
+  def inboxes_for_reply
+    replied_to_status.account.followers.inboxes
+  end
 
-    inboxes = replied_to_status.account.followers.inboxes - [@account.preferred_inbox_url]
+  def forward!
+    inboxes = inboxes_for_reblogs
+    inboxes += inboxes_for_reply if reply_to_local?
+    inboxes -= [@account.preferred_inbox_url]
 
-    ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes) do |inbox_url|
-      [payload, replied_to_status.account_id, inbox_url]
+    sender_id = reply_to_local? ? replied_to_status.account_id : rebloggers_ids.first
+
+    ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes.uniq) do |inbox_url|
+      [payload, sender_id, inbox_url]
     end
   end
 
@@ -75,8 +83,4 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
   def payload
     @payload ||= Oj.dump(@json)
   end
-
-  def lock_options
-    { redis: Redis.current, key: "create:#{object_uri}" }
-  end
 end

app/lib/activitypub/activity/flag.rb

@@ -10,6 +10,8 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity
     target_accounts.each do |target_account|
       target_statuses = target_statuses_by_account[target_account.id]
 
+      next if target_account.suspended?
+
       ReportService.new.call(
         @account,
         target_account,

app/lib/activitypub/activity/follow.rb

@@ -6,7 +6,14 @@ class ActivityPub::Activity::Follow < ActivityPub::Activity
   def perform
     target_account = account_from_uri(object_uri)
 
-    return if target_account.nil? || !target_account.local? || delete_arrived_first?(@json['id']) || @account.requested?(target_account)
+    return if target_account.nil? || !target_account.local? || delete_arrived_first?(@json['id'])
+
+    # Update id of already-existing follow requests
+    existing_follow_request = ::FollowRequest.find_by(account: @account, target_account: target_account)
+    unless existing_follow_request.nil?
+      existing_follow_request.update!(uri: @json['id'])
+      return
+    end
 
     if target_account.blocking?(@account) || target_account.domain_blocking?(@account.domain) || target_account.moved? || target_account.instance_actor?
       reject_follow_request!(target_account)
@@ -14,7 +21,9 @@ class ActivityPub::Activity::Follow < ActivityPub::Activity
     end
 
     # Fast-forward repeat follow requests
-    if @account.following?(target_account)
+    existing_follow = ::Follow.find_by(account: @account, target_account: target_account)
+    unless existing_follow.nil?
+      existing_follow.update!(uri: @json['id'])
       AuthorizeFollowService.new.call(@account, target_account, skip_follow_request: true, follow_request_uri: @json['id'])
       return
     end

app/lib/activitypub/activity/move.rb

@@ -4,9 +4,8 @@ class ActivityPub::Activity::Move < ActivityPub::Activity
   PROCESSING_COOLDOWN = 7.days.seconds
 
   def perform
-    return if origin_account.uri != object_uri || processed?
-
-    mark_as_processing!
+    return if origin_account.uri != object_uri
+    return unless mark_as_processing!
 
     target_account = ActivityPub::FetchRemoteAccountService.new.call(target_uri)
 
@@ -35,12 +34,8 @@ class ActivityPub::Activity::Move < ActivityPub::Activity
     value_or_id(@json['target'])
   end
 
-  def processed?
-    redis.exists?("move_in_progress:#{@account.id}")
-  end
-
   def mark_as_processing!
-    redis.setex("move_in_progress:#{@account.id}", PROCESSING_COOLDOWN, true)
+    redis.set("move_in_progress:#{@account.id}", true, nx: true, ex: PROCESSING_COOLDOWN)
   end
 
   def unmark_as_processing!

app/lib/activitypub/adapter.rb

@@ -1,30 +1,7 @@
 # frozen_string_literal: true
 
 class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
-  NAMED_CONTEXT_MAP = {
-    activitystreams: 'https://www.w3.org/ns/activitystreams',
-    security: 'https://w3id.org/security/v1',
-  }.freeze
-
-  CONTEXT_EXTENSION_MAP = {
-    manually_approves_followers: { 'manuallyApprovesFollowers' => 'as:manuallyApprovesFollowers' },
-    sensitive: { 'sensitive' => 'as:sensitive' },
-    hashtag: { 'Hashtag' => 'as:Hashtag' },
-    moved_to: { 'movedTo' => { '@id' => 'as:movedTo', '@type' => '@id' } },
-    also_known_as: { 'alsoKnownAs' => { '@id' => 'as:alsoKnownAs', '@type' => '@id' } },
-    emoji: { 'toot' => 'http://joinmastodon.org/ns#', 'Emoji' => 'toot:Emoji' },
-    featured: { 'toot' => 'http://joinmastodon.org/ns#', 'featured' => { '@id' => 'toot:featured', '@type' => '@id' }, 'featuredTags' => { '@id' => 'toot:featuredTags', '@type' => '@id' } },
-    property_value: { 'schema' => 'http://schema.org#', 'PropertyValue' => 'schema:PropertyValue', 'value' => 'schema:value' },
-    atom_uri: { 'ostatus' => 'http://ostatus.org#', 'atomUri' => 'ostatus:atomUri' },
-    conversation: { 'ostatus' => 'http://ostatus.org#', 'inReplyToAtomUri' => 'ostatus:inReplyToAtomUri', 'conversation' => 'ostatus:conversation' },
-    focal_point: { 'toot' => 'http://joinmastodon.org/ns#', 'focalPoint' => { '@container' => '@list', '@id' => 'toot:focalPoint' } },
-    identity_proof: { 'toot' => 'http://joinmastodon.org/ns#', 'IdentityProof' => 'toot:IdentityProof' },
-    blurhash: { 'toot' => 'http://joinmastodon.org/ns#', 'blurhash' => 'toot:blurhash' },
-    discoverable: { 'toot' => 'http://joinmastodon.org/ns#', 'discoverable' => 'toot:discoverable' },
-    voters_count: { 'toot' => 'http://joinmastodon.org/ns#', 'votersCount' => 'toot:votersCount' },
-    olm: { 'toot' => 'http://joinmastodon.org/ns#', 'Device' => 'toot:Device', 'Ed25519Signature' => 'toot:Ed25519Signature', 'Ed25519Key' => 'toot:Ed25519Key', 'Curve25519Key' => 'toot:Curve25519Key', 'EncryptedMessage' => 'toot:EncryptedMessage', 'publicKeyBase64' => 'toot:publicKeyBase64', 'deviceId' => 'toot:deviceId', 'claim' => { '@type' => '@id', '@id' => 'toot:claim' }, 'fingerprintKey' => { '@type' => '@id', '@id' => 'toot:fingerprintKey' }, 'identityKey' => { '@type' => '@id', '@id' => 'toot:identityKey' }, 'devices' => { '@type' => '@id', '@id' => 'toot:devices' }, 'messageFranking' => 'toot:messageFranking', 'messageType' => 'toot:messageType', 'cipherText' => 'toot:cipherText' },
-    suspended: { 'toot' => 'http://joinmastodon.org/ns#', 'suspended' => 'toot:suspended' },
-  }.freeze
+  include ContextHelper
 
   def self.default_key_transform
     :camel_lower
@@ -35,7 +12,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
   end
 
   def serializable_hash(options = nil)
-    named_contexts     = {}
+    named_contexts     = { activitystreams: NAMED_CONTEXT_MAP['activitystreams'] }
     context_extensions = {}
 
     options         = serialization_options(options)
@@ -45,29 +22,4 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
 
     { '@context' => serialized_context(named_contexts, context_extensions) }.merge(serialized_hash)
   end
-
-  private
-
-  def serialized_context(named_contexts_map, context_extensions_map)
-    context_array = []
-
-    named_contexts     = [:activitystreams] + named_contexts_map.keys
-    context_extensions = context_extensions_map.keys
-
-    named_contexts.each do |key|
-      context_array << NAMED_CONTEXT_MAP[key]
-    end
-
-    extensions = context_extensions.each_with_object({}) do |key, h|
-      h.merge!(CONTEXT_EXTENSION_MAP[key])
-    end
-
-    context_array << extensions unless extensions.empty?
-
-    if context_array.size == 1
-      context_array.first
-    else
-      context_array
-    end
-  end
 end

app/lib/activitypub/tag_manager.rb

@@ -12,6 +12,10 @@ class ActivityPub::TagManager
     public: 'https://www.w3.org/ns/activitystreams#Public',
   }.freeze
 
+  def public_collection?(uri)
+    uri == COLLECTIONS[:public] || uri == 'as:Public' || uri == 'Public'
+  end
+
   def url_for(target)
     return target.url if target.respond_to?(:local?) && !target.local?
 
@@ -60,6 +64,10 @@ class ActivityPub::TagManager
     account_status_replies_url(target.account, target, page_params)
   end
 
+  def followers_uri_for(target)
+    target.local? ? account_followers_url(target) : target.followers_url.presence
+  end
+
   # Primary audience of a status
   # Public statuses go out to primarily the public collection
   # Unlisted and private statuses go out primarily to the followers collection
@@ -76,17 +84,17 @@ class ActivityPub::TagManager
         account_ids = status.active_mentions.pluck(:account_id)
         to = status.account.followers.where(id: account_ids).each_with_object([]) do |account, result|
           result << uri_for(account)
-          result << account_followers_url(account) if account.group?
+          result << followers_uri_for(account) if account.group?
         end
         to.concat(FollowRequest.where(target_account_id: status.account_id, account_id: account_ids).each_with_object([]) do |request, result|
           result << uri_for(request.account)
-          result << account_followers_url(request.account) if request.account.group?
-        end)
+          result << followers_uri_for(request.account) if request.account.group?
+        end).compact
       else
         status.active_mentions.each_with_object([]) do |mention, result|
           result << uri_for(mention.account)
-          result << account_followers_url(mention.account) if mention.account.group?
-        end
+          result << followers_uri_for(mention.account) if mention.account.group?
+        end.compact
       end
     end
   end
@@ -114,17 +122,17 @@ class ActivityPub::TagManager
         account_ids = status.active_mentions.pluck(:account_id)
         cc.concat(status.account.followers.where(id: account_ids).each_with_object([]) do |account, result|
           result << uri_for(account)
-          result << account_followers_url(account) if account.group?
-        end)
+          result << followers_uri_for(account) if account.group?
+        end.compact)
         cc.concat(FollowRequest.where(target_account_id: status.account_id, account_id: account_ids).each_with_object([]) do |request, result|
           result << uri_for(request.account)
-          result << account_followers_url(request.account) if request.account.group?
-        end)
+          result << followers_uri_for(request.account) if request.account.group?
+        end.compact)
       else
         cc.concat(status.active_mentions.each_with_object([]) do |mention, result|
           result << uri_for(mention.account)
-          result << account_followers_url(mention.account) if mention.account.group?
-        end)
+          result << followers_uri_for(mention.account) if mention.account.group?
+        end.compact)
       end
     end
 

app/lib/application_extension.rb

@@ -4,6 +4,8 @@ module ApplicationExtension
   extend ActiveSupport::Concern
 
   included do
-    validates :website, url: true, if: :website?
+    validates :name, length: { maximum: 60 }
+    validates :website, url: true, length: { maximum: 2_000 }, if: :website?
+    validates :redirect_uri, length: { maximum: 2_000 }
   end
 end

app/lib/exceptions.rb

@@ -12,7 +12,11 @@ module Mastodon
   class RateLimitExceededError < Error; end
 
   class UnexpectedResponseError < Error
+    attr_reader :response
+
     def initialize(response = nil)
+      @response = response
+
       if response.respond_to? :uri
         super("#{response.uri} returned code #{response.code}")
       else

app/lib/feed_manager.rb

@@ -194,6 +194,36 @@ class FeedManager
     end
   end
 
+  # Clear all statuses from or mentioning target_account from a list feed
+  # @param [List] list
+  # @param [Account] target_account
+  # @return [void]
+  def clear_from_list(list, target_account)
+    timeline_key        = key(:list, list.id)
+    timeline_status_ids = redis.zrange(timeline_key, 0, -1)
+    statuses            = Status.where(id: timeline_status_ids).select(:id, :reblog_of_id, :account_id).to_a
+    reblogged_ids       = Status.where(id: statuses.map(&:reblog_of_id).compact, account: target_account).pluck(:id)
+    with_mentions_ids   = Mention.active.where(status_id: statuses.flat_map { |s| [s.id, s.reblog_of_id] }.compact, account: target_account).pluck(:status_id)
+
+    target_statuses = statuses.select do |status|
+      status.account_id == target_account.id || reblogged_ids.include?(status.reblog_of_id) || with_mentions_ids.include?(status.id) || with_mentions_ids.include?(status.reblog_of_id)
+    end
+
+    target_statuses.each do |status|
+      unpush_from_list(list, status)
+    end
+  end
+
+  # Clear all statuses from or mentioning target_account from an account's lists
+  # @param [Account] account
+  # @param [Account] target_account
+  # @return [void]
+  def clear_from_lists(account, target_account)
+    List.where(account: account).each do |list|
+      clear_from_list(list, target_account)
+    end
+  end
+
   # Populate home feed of account from scratch
   # @param [Account] account
   # @return [void]

app/lib/video_metadata_extractor.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+class VideoMetadataExtractor
+  attr_reader :duration, :bitrate, :video_codec, :audio_codec,
+              :colorspace, :width, :height, :frame_rate
+
+  def initialize(path)
+    @path     = path
+    @metadata = Oj.load(ffmpeg_command_output, mode: :strict, symbol_keys: true)
+
+    parse_metadata
+  rescue Terrapin::ExitStatusError, Oj::ParseError
+    @invalid = true
+  rescue Terrapin::CommandNotFoundError
+    raise Paperclip::Errors::CommandNotFoundError, 'Could not run the `ffprobe` command. Please install ffmpeg.'
+  end
+
+  def valid?
+    !@invalid
+  end
+
+  private
+
+  def ffmpeg_command_output
+    command = Terrapin::CommandLine.new('ffprobe', '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel')
+    command.run(path: @path, format: 'json', loglevel: 'fatal')
+  end
+
+  def parse_metadata
+    if @metadata.key?(:format)
+      @duration = @metadata[:format][:duration].to_f
+      @bitrate  = @metadata[:format][:bit_rate].to_i
+    end
+
+    if @metadata.key?(:streams)
+      video_streams = @metadata[:streams].select { |stream| stream[:codec_type] == 'video' }
+      audio_streams = @metadata[:streams].select { |stream| stream[:codec_type] == 'audio' }
+
+      if (video_stream = video_streams.first)
+        @video_codec = video_stream[:codec_name]
+        @colorspace  = video_stream[:pix_fmt]
+        @width       = video_stream[:width]
+        @height      = video_stream[:height]
+        @frame_rate  = video_stream[:avg_frame_rate] == '0/0' ? nil : Rational(video_stream[:avg_frame_rate])
+      end
+
+      if (audio_stream = audio_streams.first)
+        @audio_codec = audio_stream[:codec_name]
+      end
+    end
+
+    @invalid = true if @metadata.key?(:error)
+  end
+end

app/lib/webfinger.rb

@@ -46,7 +46,9 @@ class Webfinger
   def body_from_webfinger(url = standard_url, use_fallback = true)
     webfinger_request(url).perform do |res|
       if res.code == 200
-        res.body_with_limit
+        body = res.body_with_limit
+        raise Webfinger::Error, "Request for #{@uri} returned empty response" if body.empty?
+        body
       elsif res.code == 404 && use_fallback
         body_from_host_meta
       elsif res.code == 410

app/models/account.rb

@@ -55,8 +55,9 @@
 #
 
 class Account < ApplicationRecord
-  USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i
-  MENTION_RE  = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[a-z0-9]+)?)/i
+  USERNAME_RE   = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i
+  MENTION_RE    = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[[:word:]]+)?)/i
+  URL_PREFIX_RE = /\Ahttp(s?):\/\/[^\/]+/
 
   include AccountAssociations
   include AccountAvatar
@@ -301,7 +302,11 @@ class Account < ApplicationRecord
   end
 
   def fields
-    (self[:fields] || []).map { |f| Field.new(self, f) }
+    (self[:fields] || []).map do |f|
+      Field.new(self, f)
+    rescue
+      nil
+    end.compact
   end
 
   def fields_attributes=(attributes)
@@ -381,7 +386,7 @@ class Account < ApplicationRecord
   def synchronization_uri_prefix
     return 'local' if local?
 
-    @synchronization_uri_prefix ||= uri[/http(s?):\/\/[^\/]+\//]
+    @synchronization_uri_prefix ||= "#{uri[URL_PREFIX_RE]}/"
   end
 
   class Field < ActiveModelSerializers::Model
@@ -435,6 +440,9 @@ class Account < ApplicationRecord
   end
 
   class << self
+    DISALLOWED_TSQUERY_CHARACTERS = /['?\\:‘’]/.freeze
+    TEXTSEARCH = "(setweight(to_tsvector('simple', accounts.display_name), 'A') || setweight(to_tsvector('simple', accounts.username), 'B') || setweight(to_tsvector('simple', coalesce(accounts.domain, '')), 'C'))"
+
     def readonly_attributes
       super - %w(statuses_count following_count followers_count)
     end
@@ -445,70 +453,29 @@ class Account < ApplicationRecord
     end
 
     def search_for(terms, limit = 10, offset = 0)
-      textsearch, query = generate_query_for_search(terms)
+      tsquery = generate_query_for_search(terms)
 
       sql = <<-SQL.squish
         SELECT
           accounts.*,
-          ts_rank_cd(#{textsearch}, #{query}, 32) AS rank
+          ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank
         FROM accounts
-        WHERE #{query} @@ #{textsearch}
+        WHERE to_tsquery('simple', :tsquery) @@ #{TEXTSEARCH}
           AND accounts.suspended_at IS NULL
           AND accounts.moved_to_account_id IS NULL
         ORDER BY rank DESC
-        LIMIT ? OFFSET ?
+        LIMIT :limit OFFSET :offset
       SQL
 
-      records = find_by_sql([sql, limit, offset])
+      records = find_by_sql([sql, limit: limit, offset: offset, tsquery: tsquery])
       ActiveRecord::Associations::Preloader.new.preload(records, :account_stat)
       records
     end
 
     def advanced_search_for(terms, account, limit = 10, following = false, offset = 0)
-      textsearch, query = generate_query_for_search(terms)
-
-      if following
-        sql = <<-SQL.squish
-          WITH first_degree AS (
-            SELECT target_account_id
-            FROM follows
-            WHERE account_id = ?
-            UNION ALL
-            SELECT ?
-          )
-          SELECT
-            accounts.*,
-            (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank
-          FROM accounts
-          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?)
-          WHERE accounts.id IN (SELECT * FROM first_degree)
-            AND #{query} @@ #{textsearch}
-            AND accounts.suspended_at IS NULL
-            AND accounts.moved_to_account_id IS NULL
-          GROUP BY accounts.id
-          ORDER BY rank DESC
-          LIMIT ? OFFSET ?
-        SQL
-
-        records = find_by_sql([sql, account.id, account.id, account.id, limit, offset])
-      else
-        sql = <<-SQL.squish
-          SELECT
-            accounts.*,
-            (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank
-          FROM accounts
-          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?) OR (accounts.id = f.target_account_id AND f.account_id = ?)
-          WHERE #{query} @@ #{textsearch}
-            AND accounts.suspended_at IS NULL
-            AND accounts.moved_to_account_id IS NULL
-          GROUP BY accounts.id
-          ORDER BY rank DESC
-          LIMIT ? OFFSET ?
-        SQL
-
-        records = find_by_sql([sql, account.id, account.id, limit, offset])
-      end
-
+      tsquery = generate_query_for_search(terms)
+      sql = advanced_search_for_sql_template(following)
+      records = find_by_sql([sql, id: account.id, limit: limit, offset: offset, tsquery: tsquery])
       ActiveRecord::Associations::Preloader.new.preload(records, :account_stat)
       records
     end
@@ -530,12 +497,55 @@ class Account < ApplicationRecord
 
     private
 
-    def generate_query_for_search(terms)
-      terms      = Arel.sql(connection.quote(terms.gsub(/['?\\:]/, ' ')))
-      textsearch = "(setweight(to_tsvector('simple', accounts.display_name), 'A') || setweight(to_tsvector('simple', accounts.username), 'B') || setweight(to_tsvector('simple', coalesce(accounts.domain, '')), 'C'))"
-      query      = "to_tsquery('simple', ''' ' || #{terms} || ' ''' || ':*')"
+    def generate_query_for_search(unsanitized_terms)
+      terms = unsanitized_terms.gsub(DISALLOWED_TSQUERY_CHARACTERS, ' ')
 
-      [textsearch, query]
+      # The final ":*" is for prefix search.
+      # The trailing space does not seem to fit any purpose, but `to_tsquery`
+      # behaves differently with and without a leading space if the terms start
+      # with `./`, `../`, or `.. `. I don't understand why, so, in doubt, keep
+      # the same query.
+      "' #{terms} ':*"
+    end
+
+    def advanced_search_for_sql_template(following)
+      if following
+        <<-SQL.squish
+          WITH first_degree AS (
+            SELECT target_account_id
+            FROM follows
+            WHERE account_id = :id
+            UNION ALL
+            SELECT :id
+          )
+          SELECT
+            accounts.*,
+            (count(f.id) + 1) * ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank
+          FROM accounts
+          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = :id)
+          WHERE accounts.id IN (SELECT * FROM first_degree)
+            AND to_tsquery('simple', :tsquery) @@ #{TEXTSEARCH}
+            AND accounts.suspended_at IS NULL
+            AND accounts.moved_to_account_id IS NULL
+          GROUP BY accounts.id
+          ORDER BY rank DESC
+          LIMIT :limit OFFSET :offset
+        SQL
+      else
+        <<-SQL.squish
+          SELECT
+            accounts.*,
+            (count(f.id) + 1) * ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank
+          FROM accounts
+          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = :id) OR (accounts.id = f.target_account_id AND f.account_id = :id)
+          WHERE to_tsquery('simple', :tsquery) @@ #{TEXTSEARCH}
+            AND accounts.suspended_at IS NULL
+            AND accounts.moved_to_account_id IS NULL
+          GROUP BY accounts.id
+          ORDER BY rank DESC
+          LIMIT :limit OFFSET :offset
+        SQL
+      end
     end
   end
 

app/models/account_migration.rb

@@ -14,6 +14,8 @@
 #
 
 class AccountMigration < ApplicationRecord
+  include Redisable
+
   COOLDOWN_PERIOD = 30.days.freeze
 
   belongs_to :account
@@ -39,7 +41,13 @@ class AccountMigration < ApplicationRecord
 
     return false unless errors.empty?
 
-    save
+    RedisLock.acquire(lock_options) do |lock|
+      if lock.acquired?
+        save
+      else
+        raise Mastodon::RaceConditionError
+      end
+    end
   end
 
   def cooldown_at
@@ -75,4 +83,8 @@ class AccountMigration < ApplicationRecord
   def validate_migration_cooldown
     errors.add(:base, I18n.t('migrations.errors.on_cooldown')) if account.migrations.within_cooldown.exists?
   end
+
+  def lock_options
+    { redis: redis, key: "account_migration:#{account.id}" }
+  end
 end

app/models/account_note.rb

@@ -17,4 +17,5 @@ class AccountNote < ApplicationRecord
   belongs_to :target_account, class_name: 'Account'
 
   validates :account_id, uniqueness: { scope: :target_account_id }
+  validates :comment, length: { maximum: 2_000 }
 end

app/models/concerns/account_finder_concern.rb

@@ -14,6 +14,8 @@ module AccountFinderConcern
 
     def representative
       Account.find(-99)
+    rescue ActiveRecord::RecordNotFound
+      Account.create!(id: -99, actor_type: 'Application', locked: true, username: Rails.configuration.x.local_domain)
     end
 
     def find_local(username)

app/models/concerns/account_interactions.rb

@@ -243,10 +243,13 @@ module AccountInteractions
          .where('users.current_sign_in_at > ?', User::ACTIVE_DURATION.ago)
   end
 
-  def remote_followers_hash(url_prefix)
-    Rails.cache.fetch("followers_hash:#{id}:#{url_prefix}") do
+  def remote_followers_hash(url)
+    url_prefix = url[Account::URL_PREFIX_RE]
+    return if url_prefix.blank?
+
+    Rails.cache.fetch("followers_hash:#{id}:#{url_prefix}/") do
       digest = "\x00" * 32
-      followers.where(Account.arel_table[:uri].matches(url_prefix + '%', false, true)).pluck_each(:uri) do |uri|
+      followers.where(Account.arel_table[:uri].matches("#{Account.sanitize_sql_like(url_prefix)}/%", false, true)).or(followers.where(uri: url_prefix)).pluck_each(:uri) do |uri|
         Xorcist.xor!(digest, Digest::SHA256.digest(uri))
       end
       digest.unpack('H*')[0]

app/models/concerns/expireable.rb

@@ -17,7 +17,7 @@ module Expireable
     end
 
     def expires_in=(interval)
-      self.expires_at = interval.to_i.seconds.from_now if interval.present?
+      self.expires_at = interval.present? ? interval.to_i.seconds.from_now : nil 
       @expires_in     = interval
     end
 

app/models/concerns/omniauthable.rb

@@ -68,7 +68,6 @@ module Omniauthable
     def user_params_from_auth(email, auth)
       {
         email: email || "#{TEMP_EMAIL_PREFIX}-#{auth.uid}-#{auth.provider}.com",
-        password: Devise.friendly_token[0, 20],
         agreement: true,
         external: true,
         account_attributes: {

app/models/follow_request.rb

@@ -29,7 +29,7 @@ class FollowRequest < ApplicationRecord
   validates :account_id, uniqueness: { scope: :target_account_id }
 
   def authorize!
-    account.follow!(target_account, reblogs: show_reblogs, notify: notify, uri: uri)
+    account.follow!(target_account, reblogs: show_reblogs, notify: notify, uri: uri, bypass_limit: true)
     MergeWorker.perform_async(target_account.id, account.id) if account.local?
     destroy!
   end

app/models/media_attachment.rb

@@ -287,7 +287,7 @@ class MediaAttachment < ApplicationRecord
       if instance.file_content_type == 'image/gif'
         [:gif_transcoder, :blurhash_transcoder]
       elsif VIDEO_MIME_TYPES.include?(instance.file_content_type)
-        [:video_transcoder, :blurhash_transcoder, :type_corrector]
+        [:transcoder, :blurhash_transcoder, :type_corrector]
       elsif AUDIO_MIME_TYPES.include?(instance.file_content_type)
         [:image_extractor, :transcoder, :type_corrector]
       else
@@ -388,7 +388,7 @@ class MediaAttachment < ApplicationRecord
   # paths but ultimately the same file, so it makes sense to memoize the
   # result while disregarding the path
   def ffmpeg_data(path = nil)
-    @ffmpeg_data ||= FFMPEG::Movie.new(path)
+    @ffmpeg_data ||= VideoMetadataExtractor.new(path)
   end
 
   def enqueue_processing

app/models/status.rb

@@ -96,15 +96,12 @@ class Status < ApplicationRecord
   scope :not_excluded_by_account, ->(account) { where.not(account_id: account.excluded_from_timeline_account_ids) }
   scope :not_domain_blocked_by_account, ->(account) { account.excluded_from_timeline_domains.blank? ? left_outer_joins(:account) : left_outer_joins(:account).where('accounts.domain IS NULL OR accounts.domain NOT IN (?)', account.excluded_from_timeline_domains) }
   scope :tagged_with_all, ->(tag_ids) {
-    Array(tag_ids).reduce(self) do |result, id|
+    Array(tag_ids).map(&:to_i).reduce(self) do |result, id|
       result.joins("INNER JOIN statuses_tags t#{id} ON t#{id}.status_id = statuses.id AND t#{id}.tag_id = #{id}")
     end
   }
   scope :tagged_with_none, ->(tag_ids) {
-    Array(tag_ids).reduce(self) do |result, id|
-      result.joins("LEFT OUTER JOIN statuses_tags t#{id} ON t#{id}.status_id = statuses.id AND t#{id}.tag_id = #{id}")
-            .where("t#{id}.tag_id IS NULL")
-    end
+    where('NOT EXISTS (SELECT * FROM statuses_tags forbidden WHERE forbidden.status_id = statuses.id AND forbidden.tag_id IN (?))', tag_ids)
   }
 
   cache_associated :application,
@@ -114,7 +111,7 @@ class Status < ApplicationRecord
                    :tags,
                    :preview_cards,
                    :preloadable_poll,
-                   account: :account_stat,
+                   account: [:account_stat, :user],
                    active_mentions: { account: :account_stat },
                    reblog: [
                      :application,
@@ -124,7 +121,7 @@ class Status < ApplicationRecord
                      :conversation,
                      :status_stat,
                      :preloadable_poll,
-                     account: :account_stat,
+                     account: [:account_stat, :user],
                      active_mentions: { account: :account_stat },
                    ],
                    thread: { account: :account_stat }
@@ -301,7 +298,7 @@ class Status < ApplicationRecord
 
       return if account_ids.empty?
 
-      accounts = Account.where(id: account_ids).includes(:account_stat).each_with_object({}) { |a, h| h[a.id] = a }
+      accounts = Account.where(id: account_ids).includes(:account_stat, :user).index_by(&:id)
 
       cached_items.each do |item|
         item.account = accounts[item.account_id]
@@ -422,7 +419,7 @@ class Status < ApplicationRecord
   end
 
   def decrement_counter_caches
-    return if direct_visibility?
+    return if direct_visibility? || new_record?
 
     account&.decrement_count!(:statuses_count)
     reblog&.decrement_count!(:reblogs_count) if reblog?

app/models/user.rb

@@ -63,7 +63,7 @@ class User < ApplicationRecord
   devise :two_factor_backupable,
          otp_number_of_backup_codes: 10
 
-  devise :registerable, :recoverable, :rememberable, :validatable,
+  devise :registerable, :recoverable, :validatable,
          :confirmable
 
   include Omniauthable
@@ -86,11 +86,11 @@ class User < ApplicationRecord
   validates :invite_request, presence: true, on: :create, if: :invite_text_required?
 
   validates :locale, inclusion: I18n.available_locales.map(&:to_s), if: :locale?
-  validates_with BlacklistedEmailValidator, on: :create
+  validates_with BlacklistedEmailValidator, if: -> { !confirmed? }
   validates_with EmailMxValidator, if: :validate_email_dns?
   validates :agreement, acceptance: { allow_nil: false, accept: [true, 'true', '1'] }, on: :create
 
-  # Those are honeypot/antispam fields
+  # Honeypot/anti-spam fields
   attr_accessor :registration_form_time, :website, :confirm_password
 
   validates_with RegistrationFormTimeValidator, on: :create
@@ -152,7 +152,7 @@ class User < ApplicationRecord
 
   def confirm
     new_user      = !confirmed?
-    self.approved = true if open_registrations?
+    self.approved = true if open_registrations? && !sign_up_from_ip_requires_approval?
 
     super
 
@@ -468,7 +468,7 @@ class User < ApplicationRecord
   end
 
   def validate_email_dns?
-    email_changed? && !(Rails.env.test? || Rails.env.development?)
+    email_changed? && !external? && !(Rails.env.test? || Rails.env.development?)
   end
 
   def invite_text_required?

app/serializers/manifest_serializer.rb

@@ -48,7 +48,7 @@ class ManifestSerializer < ActiveModel::Serializer
   end
 
   def scope
-    root_url
+    '/'
   end
 
   def share_target

app/services/activitypub/process_account_service.rb

@@ -281,7 +281,7 @@ class ActivityPub::ProcessAccountService < BaseService
   end
 
   def lock_options
-    { redis: Redis.current, key: "process_account:#{@uri}" }
+    { redis: Redis.current, key: "process_account:#{@uri}", autorelease: 15.minutes.seconds }
   end
 
   def process_tags

app/services/activitypub/process_collection_service.rb

@@ -5,11 +5,27 @@ class ActivityPub::ProcessCollectionService < BaseService
 
   def call(body, account, **options)
     @account = account
-    @json    = Oj.load(body, mode: :strict)
+    @json    = original_json = Oj.load(body, mode: :strict)
     @options = options
 
+    begin
+      @json = compact(@json) if @json['signature'].is_a?(Hash)
+    rescue JSON::LD::JsonLdError => e
+      Rails.logger.debug "Error when compacting JSON-LD document for #{value_or_id(@json['actor'])}: #{e.message}"
+      @json = original_json.without('signature')
+    end
+
     return if !supported_context? || (different_actor? && verify_account!.nil?) || suspended_actor? || @account.local?
 
+    if @json['signature'].present?
+      # We have verified the signature, but in the compaction step above, might
+      # have introduced incompatibilities with other servers that do not
+      # normalize the JSON-LD documents (for instance, previous Mastodon
+      # versions), so skip redistribution if we can't get a safe document.
+      patch_for_forwarding!(original_json, @json)
+      @json.delete('signature') unless safe_for_forwarding?(original_json, @json)
+    end
+
     case @json['type']
     when 'Collection', 'CollectionPage'
       process_items @json['items']

app/services/after_block_service.rb

@@ -6,6 +6,7 @@ class AfterBlockService < BaseService
     @target_account = target_account
 
     clear_home_feed!
+    clear_list_feeds!
     clear_notifications!
     clear_conversations!
   end
@@ -16,6 +17,10 @@ class AfterBlockService < BaseService
     FeedManager.instance.clear_from_home(@account, @target_account)
   end
 
+  def clear_list_feeds!
+    FeedManager.instance.clear_from_lists(@account, @target_account)
+  end
+
   def clear_conversations!
     AccountConversation.where(account: @account).where('? = ANY(participant_account_ids)', @target_account.id).in_batches.destroy_all
   end

app/services/fetch_link_card_service.rb

@@ -174,6 +174,6 @@ class FetchLinkCardService < BaseService
   end
 
   def lock_options
-    { redis: Redis.current, key: "fetch:#{@url}" }
+    { redis: Redis.current, key: "fetch:#{@url}", autorelease: 15.minutes.seconds }
   end
 end

app/services/fetch_oembed_service.rb

@@ -2,6 +2,7 @@
 
 class FetchOEmbedService
   ENDPOINT_CACHE_EXPIRES_IN = 24.hours.freeze
+  URL_REGEX                 = /(=(http[s]?(%3A|:)(\/\/|%2F%2F)))([^&]*)/i.freeze
 
   attr_reader :url, :options, :format, :endpoint_url
 
@@ -55,10 +56,12 @@ class FetchOEmbedService
   end
 
   def cache_endpoint!
+    return unless URL_REGEX.match?(@endpoint_url)
+
     url_domain = Addressable::URI.parse(@url).normalized_host
 
     endpoint_hash = {
-      endpoint: @endpoint_url.gsub(/(=(http[s]?(%3A|:)(\/\/|%2F%2F)))([^&]*)/i, '={url}'),
+      endpoint: @endpoint_url.gsub(URL_REGEX, '={url}'),
       format: @format,
     }
 

app/services/notify_service.rb

@@ -67,8 +67,53 @@ class NotifyService < BaseService
     message? && @notification.target_status.direct_visibility?
   end
 
+  # Returns true if the sender has been mentionned by the recipient up the thread
   def response_to_recipient?
-    @notification.target_status.in_reply_to_account_id == @recipient.id && @notification.target_status.thread&.direct_visibility?
+    return false if @notification.target_status.in_reply_to_id.nil?
+
+    # Using an SQL CTE to avoid unneeded back-and-forth with SQL server in case of long threads
+    !Status.count_by_sql([<<-SQL.squish, id: @notification.target_status.in_reply_to_id, recipient_id: @recipient.id, sender_id: @notification.from_account.id]).zero?
+      WITH RECURSIVE ancestors(id, in_reply_to_id, replying_to_sender, path) AS (
+          SELECT
+            s.id,
+            s.in_reply_to_id,
+            (CASE
+              WHEN s.account_id = :recipient_id THEN
+                EXISTS (
+                  SELECT *
+                  FROM mentions m
+                  WHERE m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
+                )
+              ELSE
+                FALSE
+             END),
+            ARRAY[s.id]
+          FROM statuses s
+          WHERE s.id = :id
+        UNION ALL
+          SELECT
+            s.id,
+            s.in_reply_to_id,
+            (CASE
+              WHEN s.account_id = :recipient_id THEN
+                EXISTS (
+                  SELECT *
+                  FROM mentions m
+                  WHERE m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
+                )
+              ELSE
+                FALSE
+             END),
+            st.path || s.id
+          FROM ancestors st
+          JOIN statuses s ON s.id = st.in_reply_to_id
+          WHERE st.replying_to_sender IS FALSE AND NOT s.id = ANY(path)
+      )
+      SELECT COUNT(*)
+      FROM ancestors st
+      JOIN statuses s ON s.id = st.id
+      WHERE st.replying_to_sender IS TRUE AND s.visibility = 3
+    SQL
   end
 
   def from_staff?

app/services/post_status_service.rb

@@ -74,6 +74,9 @@ class PostStatusService < BaseService
     status_for_validation = @account.statuses.build(status_attributes)
 
     if status_for_validation.valid?
+      # Marking the status as destroyed is necessary to prevent the status from being
+      # persisted when the associated media attachments get updated when creating the
+      # scheduled status.
       status_for_validation.destroy
 
       # The following transaction block is needed to wrap the UPDATEs to

app/services/remove_status_service.rb

@@ -16,6 +16,8 @@ class RemoveStatusService < BaseService
     @account  = status.account
     @options  = options
 
+    @status.discard
+
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
         remove_from_self if @account.local?
@@ -168,6 +170,6 @@ class RemoveStatusService < BaseService
   end
 
   def lock_options
-    { redis: Redis.current, key: "distribute:#{@status.id}" }
+    { redis: Redis.current, key: "distribute:#{@status.id}", autorelease: 5.minutes.seconds }
   end
 end

app/services/report_service.rb

@@ -10,6 +10,8 @@ class ReportService < BaseService
     @comment        = options.delete(:comment) || ''
     @options        = options
 
+    raise ActiveRecord::RecordNotFound if @target_account.suspended?
+
     create_report!
     notify_staff!
     forward_to_origin! if !@target_account.local? && ActiveModel::Type::Boolean.new.cast(@options[:forward])

app/services/resolve_account_service.rb

@@ -141,10 +141,11 @@ class ResolveAccountService < BaseService
   end
 
   def queue_deletion!
+    @account.suspend!(origin: :remote)
     AccountDeletionWorker.perform_async(@account.id, reserve_username: false, skip_activitypub: true)
   end
 
   def lock_options
-    { redis: Redis.current, key: "resolve:#{@username}@#{@domain}" }
+    { redis: Redis.current, key: "resolve:#{@username}@#{@domain}", autorelease: 15.minutes.seconds }
   end
 end

app/services/suspend_account_service.rb

@@ -42,7 +42,13 @@ class SuspendAccountService < BaseService
   end
 
   def distribute_update_actor!
-    ActivityPub::UpdateDistributionWorker.perform_async(@account.id) if @account.local?
+    return unless @account.local?
+
+    account_reach_finder = AccountReachFinder.new(@account)
+
+    ActivityPub::DeliveryWorker.push_bulk(account_reach_finder.inboxes) do |inbox_url|
+      [signed_activity_json, @account.id, inbox_url]
+    end
   end
 
   def unmerge_from_home_timelines!
@@ -90,4 +96,8 @@ class SuspendAccountService < BaseService
       end
     end
   end
+
+  def signed_activity_json
+    @signed_activity_json ||= Oj.dump(serialize_payload(@account, ActivityPub::UpdateSerializer, signer: @account))
+  end
 end

app/services/unsuspend_account_service.rb

@@ -7,11 +7,12 @@ class UnsuspendAccountService < BaseService
     unsuspend!
     refresh_remote_account!
 
-    return if @account.nil?
+    return if @account.nil? || @account.suspended?
 
     merge_into_home_timelines!
     merge_into_list_timelines!
     publish_media_attachments!
+    distribute_update_actor!
   end
 
   private
@@ -36,6 +37,16 @@ class UnsuspendAccountService < BaseService
     # @account would now be nil.
   end
 
+  def distribute_update_actor!
+    return unless @account.local?
+
+    account_reach_finder = AccountReachFinder.new(@account)
+
+    ActivityPub::DeliveryWorker.push_bulk(account_reach_finder.inboxes) do |inbox_url|
+      [signed_activity_json, @account.id, inbox_url]
+    end
+  end
+
   def merge_into_home_timelines!
     @account.followers_for_local_distribution.find_each do |follower|
       FeedManager.instance.merge_into_home(@account, follower)
@@ -81,4 +92,8 @@ class UnsuspendAccountService < BaseService
       end
     end
   end
+
+  def signed_activity_json
+    @signed_activity_json ||= Oj.dump(serialize_payload(@account, ActivityPub::UpdateSerializer, signer: @account))
+  end
 end

app/views/admin/reports/show.html.haml

@@ -92,7 +92,7 @@
 %hr.spacer
 
 .speech-bubble
-  .speech-bubble__bubble= simple_format(@report.comment.presence || t('admin.reports.comment.none'))
+  .speech-bubble__bubble= simple_format(h(@report.comment.presence || t('admin.reports.comment.none')))
   .speech-bubble__owner
     - if @report.account.local?
       = admin_account_link_to @report.account

app/views/filters/_fields.html.haml

@@ -2,7 +2,7 @@
   .fields-row__column.fields-row__column-6.fields-group
     = f.input :phrase, as: :string, wrapper: :with_label, hint: false
   .fields-row__column.fields-row__column-6.fields-group
-    = f.input :expires_in, wrapper: :with_label, collection: [30.minutes, 1.hour, 6.hours, 12.hours, 1.day, 1.week].map(&:to_i), label_method: lambda { |i| I18n.t("invites.expires_in.#{i}") }, prompt: I18n.t('invites.expires_in_prompt')
+    = f.input :expires_in, wrapper: :with_label, collection: [30.minutes, 1.hour, 6.hours, 12.hours, 1.day, 1.week].map(&:to_i), label_method: lambda { |i| I18n.t("invites.expires_in.#{i}") }, include_blank: I18n.t('invites.expires_in_prompt')
 
 .fields-group
   = f.input :context, wrapper: :with_block_label, collection: CustomFilter::VALID_CONTEXTS, as: :check_boxes, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li', label_method: lambda { |context| I18n.t("filters.contexts.#{context}") }, include_blank: false

app/views/layouts/application.html.haml

@@ -31,7 +31,7 @@
     = stylesheet_link_tag '/inert.css', skip_pipeline: true, media: 'all', id: 'inert-style'
 
     - if Setting.custom_css.present?
-      = stylesheet_link_tag custom_css_path, media: 'all'
+      = stylesheet_link_tag custom_css_path, host: request.host, media: 'all'
 
     = yield :header_tags
 

app/workers/activitypub/delivery_worker.rb

@@ -44,11 +44,7 @@ class ActivityPub::DeliveryWorker
   end
 
   def synchronization_header
-    "collectionId=\"#{account_followers_url(@source_account)}\", digest=\"#{@source_account.remote_followers_hash(inbox_url_prefix)}\", url=\"#{account_followers_synchronization_url(@source_account)}\""
-  end
-
-  def inbox_url_prefix
-    @inbox_url[/http(s?):\/\/[^\/]+\//]
+    "collectionId=\"#{account_followers_url(@source_account)}\", digest=\"#{@source_account.remote_followers_hash(@inbox_url)}\", url=\"#{account_followers_synchronization_url(@source_account)}\""
   end
 
   def perform_request

app/workers/distribution_worker.rb

@@ -4,7 +4,7 @@ class DistributionWorker
   include Sidekiq::Worker
 
   def perform(status_id)
-    RedisLock.acquire(redis: Redis.current, key: "distribute:#{status_id}") do |lock|
+    RedisLock.acquire(redis: Redis.current, key: "distribute:#{status_id}", autorelease: 5.minutes.seconds) do |lock|
       if lock.acquired?
         FanOutOnWriteService.new.call(Status.find(status_id))
       else

app/workers/move_worker.rb

@@ -13,9 +13,13 @@ class MoveWorker
       queue_follow_unfollows!
     end
 
+    @deferred_error = nil
+
     copy_account_notes!
     carry_blocks_over!
     carry_mutes_over!
+
+    raise @deferred_error unless @deferred_error.nil?
   rescue ActiveRecord::RecordNotFound
     true
   end
@@ -36,21 +40,31 @@ class MoveWorker
 
     @source_account.followers.local.select(:id).find_in_batches do |accounts|
       UnfollowFollowWorker.push_bulk(accounts.map(&:id)) { |follower_id| [follower_id, @source_account.id, @target_account.id, bypass_locked] }
+    rescue => e
+      @deferred_error = e
     end
   end
 
   def copy_account_notes!
     AccountNote.where(target_account: @source_account).find_each do |note|
-      text = I18n.with_locale(note.account.user.locale || I18n.default_locale) do
+      text = I18n.with_locale(note.account.user&.locale || I18n.default_locale) do
         I18n.t('move_handler.copy_account_note_text', acct: @source_account.acct)
       end
 
       new_note = AccountNote.find_by(account: note.account, target_account: @target_account)
       if new_note.nil?
-        AccountNote.create!(account: note.account, target_account: @target_account, comment: [text, note.comment].join('\n'))
+        begin
+          AccountNote.create!(account: note.account, target_account: @target_account, comment: [text, note.comment].join("\n"))
+        rescue ActiveRecord::RecordInvalid
+          AccountNote.create!(account: note.account, target_account: @target_account, comment: note.comment)
+        end
       else
-        new_note.update!(comment: [text, note.comment, '\n', new_note.comment].join('\n'))
+        new_note.update!(comment: [text, note.comment, "\n", new_note.comment].join("\n"))
       end
+    rescue ActiveRecord::RecordInvalid
+      nil
+    rescue => e
+      @deferred_error = e
     end
   end
 
@@ -60,6 +74,8 @@ class MoveWorker
         BlockService.new.call(block.account, @target_account)
         add_account_note_if_needed!(block.account, 'move_handler.carry_blocks_over_text')
       end
+    rescue => e
+      @deferred_error = e
     end
   end
 
@@ -67,12 +83,14 @@ class MoveWorker
     @source_account.muted_by_relationships.where(account: Account.local).find_each do |mute|
       MuteService.new.call(mute.account, @target_account, notifications: mute.hide_notifications) unless mute.account.muting?(@target_account) || mute.account.following?(@target_account)
       add_account_note_if_needed!(mute.account, 'move_handler.carry_mutes_over_text')
+    rescue => e
+      @deferred_error = e
     end
   end
 
   def add_account_note_if_needed!(account, id)
     unless AccountNote.where(account: account, target_account: @target_account).exists?
-      text = I18n.with_locale(account.user.locale || I18n.default_locale) do
+      text = I18n.with_locale(account.user&.locale || I18n.default_locale) do
         I18n.t(id, acct: @source_account.acct)
       end
       AccountNote.create!(account: account, target_account: @target_account, comment: text)

app/workers/redownload_media_worker.rb

@@ -3,6 +3,7 @@
 class RedownloadMediaWorker
   include Sidekiq::Worker
   include ExponentialBackoff
+  include JsonLdHelper
 
   sidekiq_options queue: 'pull', retry: 3
 
@@ -15,6 +16,14 @@ class RedownloadMediaWorker
     media_attachment.download_thumbnail!
     media_attachment.save
   rescue ActiveRecord::RecordNotFound
-    true
+    # Do nothing
+  rescue Mastodon::UnexpectedResponseError => e
+    response = e.response
+
+    if response_error_unsalvageable?(response)
+      # Give up
+    else
+      raise e
+    end
   end
 end

app/workers/thread_resolve_worker.rb

@@ -14,5 +14,7 @@ class ThreadResolveWorker
 
     child_status.thread = parent_status
     child_status.save!
+  rescue ActiveRecord::RecordNotFound
+    true
   end
 end

chart/values.yaml.template

@@ -4,7 +4,7 @@ image:
   repository: tootsuite/mastodon
   pullPolicy: Always
   # https://hub.docker.com/r/tootsuite/mastodon/tags
-  tag: v3.3.0
+  tag: v3.3.2
   # alternatively, use `latest` for the latest release or `edge` for the image
   # built from the most recent commit
   #

config/application.rb

@@ -11,12 +11,12 @@ require_relative '../lib/redis/namespace_extensions'
 require_relative '../lib/paperclip/url_generator_extensions'
 require_relative '../lib/paperclip/attachment_extensions'
 require_relative '../lib/paperclip/media_type_spoof_detector_extensions'
-require_relative '../lib/paperclip/transcoder_extensions'
 require_relative '../lib/paperclip/lazy_thumbnail'
 require_relative '../lib/paperclip/gif_transcoder'
-require_relative '../lib/paperclip/video_transcoder'
+require_relative '../lib/paperclip/transcoder'
 require_relative '../lib/paperclip/type_corrector'
 require_relative '../lib/paperclip/response_with_limit_adapter'
+require_relative '../lib/terrapin/multi_pipe_extensions'
 require_relative '../lib/mastodon/snowflake'
 require_relative '../lib/mastodon/version'
 require_relative '../lib/devise/two_factor_ldap_authenticatable'

config/brakeman.ignore

@@ -153,46 +153,6 @@
       "confidence": "High",
       "note": ""
     },
-    {
-      "warning_type": "SQL Injection",
-      "warning_code": 0,
-      "fingerprint": "6e4051854bb62e2ddbc671f82d6c2328892e1134b8b28105ecba9b0122540714",
-      "check_name": "SQL",
-      "message": "Possible SQL injection",
-      "file": "app/models/account.rb",
-      "line": 491,
-      "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
-      "code": "find_by_sql([\"          WITH first_degree AS (\\n            SELECT target_account_id\\n            FROM follows\\n            WHERE account_id = ?\\n            UNION ALL\\n            SELECT ?\\n          )\\n          SELECT\\n            accounts.*,\\n            (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n          FROM accounts\\n          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?)\\n          WHERE accounts.id IN (SELECT * FROM first_degree)\\n            AND #{query} @@ #{textsearch}\\n            AND accounts.suspended_at IS NULL\\n            AND accounts.moved_to_account_id IS NULL\\n          GROUP BY accounts.id\\n          ORDER BY rank DESC\\n          LIMIT ? OFFSET ?\\n\".squish, account.id, account.id, account.id, limit, offset])",
-      "render_path": null,
-      "location": {
-        "type": "method",
-        "class": "Account",
-        "method": "advanced_search_for"
-      },
-      "user_input": "textsearch",
-      "confidence": "Medium",
-      "note": ""
-    },
-    {
-      "warning_type": "SQL Injection",
-      "warning_code": 0,
-      "fingerprint": "6f075c1484908e3ec9bed21ab7cf3c7866be8da3881485d1c82e13093aefcbd7",
-      "check_name": "SQL",
-      "message": "Possible SQL injection",
-      "file": "app/models/status.rb",
-      "line": 105,
-      "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
-      "code": "result.joins(\"LEFT OUTER JOIN statuses_tags t#{id} ON t#{id}.status_id = statuses.id AND t#{id}.tag_id = #{id}\")",
-      "render_path": null,
-      "location": {
-        "type": "method",
-        "class": "Status",
-        "method": null
-      },
-      "user_input": "id",
-      "confidence": "Weak",
-      "note": ""
-    },
     {
       "warning_type": "Mass Assignment",
       "warning_code": 105,
@@ -213,26 +173,6 @@
       "confidence": "High",
       "note": ""
     },
-    {
-      "warning_type": "SQL Injection",
-      "warning_code": 0,
-      "fingerprint": "9251d682c4e2840e1b2fea91e7d758efe2097ecb7f6255c065e3750d25eb178c",
-      "check_name": "SQL",
-      "message": "Possible SQL injection",
-      "file": "app/models/account.rb",
-      "line": 460,
-      "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
-      "code": "find_by_sql([\"        SELECT\\n          accounts.*,\\n          ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n        FROM accounts\\n        WHERE #{query} @@ #{textsearch}\\n          AND accounts.suspended_at IS NULL\\n          AND accounts.moved_to_account_id IS NULL\\n        ORDER BY rank DESC\\n        LIMIT ? OFFSET ?\\n\".squish, limit, offset])",
-      "render_path": null,
-      "location": {
-        "type": "method",
-        "class": "Account",
-        "method": "search_for"
-      },
-      "user_input": "textsearch",
-      "confidence": "Medium",
-      "note": ""
-    },
     {
       "warning_type": "SQL Injection",
       "warning_code": 0,
@@ -324,26 +264,6 @@
       "confidence": "High",
       "note": ""
     },
-    {
-      "warning_type": "SQL Injection",
-      "warning_code": 0,
-      "fingerprint": "e21d8fee7a5805761679877ca35ed1029c64c45ef3b4012a30262623e1ba8bb9",
-      "check_name": "SQL",
-      "message": "Possible SQL injection",
-      "file": "app/models/account.rb",
-      "line": 507,
-      "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
-      "code": "find_by_sql([\"          SELECT\\n            accounts.*,\\n            (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n          FROM accounts\\n          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?) OR (accounts.id = f.target_account_id AND f.account_id = ?)\\n          WHERE #{query} @@ #{textsearch}\\n            AND accounts.suspended_at IS NULL\\n            AND accounts.moved_to_account_id IS NULL\\n          GROUP BY accounts.id\\n          ORDER BY rank DESC\\n          LIMIT ? OFFSET ?\\n\".squish, account.id, account.id, limit, offset])",
-      "render_path": null,
-      "location": {
-        "type": "method",
-        "class": "Account",
-        "method": "advanced_search_for"
-      },
-      "user_input": "textsearch",
-      "confidence": "Medium",
-      "note": ""
-    },
     {
       "warning_type": "Mass Assignment",
       "warning_code": 105,

config/environments/production.rb

@@ -108,7 +108,7 @@ Rails.application.configure do
     'Server'                 => 'Mastodon',
     'X-Frame-Options'        => 'DENY',
     'X-Content-Type-Options' => 'nosniff',
-    'X-XSS-Protection'       => '1; mode=block',
+    'X-XSS-Protection'       => '0',
   }
 
   config.x.otp_secret = ENV.fetch('OTP_SECRET')

config/initializers/devise.rb

@@ -1,3 +1,5 @@
+require 'devise/strategies/authenticatable'
+
 Warden::Manager.after_set_user except: :fetch do |user, warden|
   if user.session_active?(warden.cookies.signed['_session_id'] || warden.raw_session['auth_id'])
     session_id = warden.cookies.signed['_session_id'] || warden.raw_session['auth_id']
@@ -72,17 +74,48 @@ module Devise
   mattr_accessor :ldap_uid_conversion_replace
   @@ldap_uid_conversion_replace = nil
 
-  class Strategies::PamAuthenticatable
-    def valid?
-      super && ::Devise.pam_authentication
+  module Strategies
+    class PamAuthenticatable
+      def valid?
+        super && ::Devise.pam_authentication
+      end
+    end
+
+    class SessionActivationRememberable < Authenticatable
+      def valid?
+        @session_cookie = nil
+        session_cookie.present?
+      end
+
+      def authenticate!
+        resource = SessionActivation.find_by(session_id: session_cookie)&.user
+
+        unless resource
+          cookies.delete('_session_id')
+          return pass
+        end
+
+        if validate(resource)
+          success!(resource)
+        end
+      end
+
+      private
+
+      def session_cookie
+        @session_cookie ||= cookies.signed['_session_id']
+      end
     end
   end
 end
 
+Warden::Strategies.add(:session_activation_rememberable, Devise::Strategies::SessionActivationRememberable)
+
 Devise.setup do |config|
   config.warden do |manager|
     manager.default_strategies(scope: :user).unshift :two_factor_ldap_authenticatable if Devise.ldap_authentication
     manager.default_strategies(scope: :user).unshift :two_factor_pam_authenticatable  if Devise.pam_authentication
+    manager.default_strategies(scope: :user).unshift :session_activation_rememberable
     manager.default_strategies(scope: :user).unshift :two_factor_authenticatable
     manager.default_strategies(scope: :user).unshift :two_factor_backupable
   end

config/initializers/doorkeeper.rb

@@ -52,6 +52,11 @@ Doorkeeper.configure do
   # Issue access tokens with refresh token (disabled by default)
   # use_refresh_token
 
+  # Forbids creating/updating applications with arbitrary scopes that are
+  # not in configuration, i.e. `default_scopes` or `optional_scopes`.
+  # (Disabled by default)
+  enforce_configured_scopes
+
   # Provide support for an owner to be assigned to each registered application (disabled by default)
   # Optional parameter :confirmation => true (default false) if you want to enforce ownership of
   # a registered application

config/initializers/omniauth.rb

@@ -60,6 +60,7 @@ Devise.setup do |config|
     saml_options[:attribute_statements][:verified] = [ENV['SAML_ATTRIBUTES_STATEMENTS_VERIFIED']] if ENV['SAML_ATTRIBUTES_STATEMENTS_VERIFIED']
     saml_options[:attribute_statements][:verified_email] = [ENV['SAML_ATTRIBUTES_STATEMENTS_VERIFIED_EMAIL']] if ENV['SAML_ATTRIBUTES_STATEMENTS_VERIFIED_EMAIL']
     saml_options[:uid_attribute] = ENV['SAML_UID_ATTRIBUTE'] if ENV['SAML_UID_ATTRIBUTE']
+    saml_options[:allowed_clock_drift] = ENV['SAML_ALLOWED_CLOCK_DRIFT'] if ENV['SAML_ALLOWED_CLOCK_DRIFT']
     config.omniauth :saml, saml_options
   end
 end

db/migrate/20181026034033_remove_faux_remote_account_duplicates.rb

@@ -1,6 +1,46 @@
 class RemoveFauxRemoteAccountDuplicates < ActiveRecord::Migration[5.2]
   disable_ddl_transaction!
 
+  class StreamEntry < ApplicationRecord
+    # Dummy class, to make migration possible across version changes
+    belongs_to :account, inverse_of: :stream_entries
+  end
+
+  class Status < ApplicationRecord
+    # Dummy class, to make migration possible across version changes
+    belongs_to :account, inverse_of: :statuses
+    has_many :favourites, inverse_of: :status, dependent: :destroy
+    has_many :mentions, dependent: :destroy, inverse_of: :status
+  end
+
+  class Favourite < ApplicationRecord
+    # Dummy class, to make migration possible across version changes
+    belongs_to :account, inverse_of: :favourites
+    belongs_to :status,  inverse_of: :favourites
+  end
+
+  class Mention < ApplicationRecord
+    # Dummy class, to make migration possible across version changes
+    belongs_to :account, inverse_of: :mentions
+    belongs_to :status
+  end
+
+  class Notification < ApplicationRecord
+    # Dummy class, to make migration possible across version changes
+    belongs_to :account, optional: true
+    belongs_to :from_account, class_name: 'Account', optional: true
+    belongs_to :activity, polymorphic: true, optional: true
+  end
+
+  class Account < ApplicationRecord
+    # Dummy class, to make migration possible across version changes
+    has_many :stream_entries, inverse_of: :account, dependent: :destroy
+    has_many :statuses, inverse_of: :account, dependent: :destroy
+    has_many :favourites, inverse_of: :account, dependent: :destroy
+    has_many :mentions, inverse_of: :account, dependent: :destroy
+    has_many :notifications, inverse_of: :account, dependent: :destroy
+  end
+
   def up
     local_domain = Rails.configuration.x.local_domain
 

db/migrate/20190715164535_add_instance_actor.rb

@@ -1,4 +1,9 @@
 class AddInstanceActor < ActiveRecord::Migration[5.2]
+  class Account < ApplicationRecord
+    # Dummy class, to make migration possible across version changes
+    validates :username, uniqueness: { scope: :domain, case_sensitive: false }
+  end
+
   def up
     Account.create!(id: -99, actor_type: 'Application', locked: true, username: Rails.configuration.x.local_domain)
   end

db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb

@@ -15,7 +15,13 @@ class AddCaseInsensitiveIndexToTags < ActiveRecord::Migration[5.2]
       Tag.where(id: redundant_tag_ids).in_batches.delete_all
     end
 
-    safety_assured { execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower ON tags (lower(name))' }
+    begin
+      safety_assured { execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower ON tags (lower(name))' }
+    rescue ActiveRecord::StatementInvalid
+      remove_index :tags, name: 'index_tags_on_name_lower'
+      raise
+    end
+
     remove_index :tags, name: 'index_tags_on_name'
     remove_index :tags, name: 'hashtag_search_index'
   end

db/migrate/20191007013357_update_pt_locales.rb

@@ -1,4 +1,8 @@
 class UpdatePtLocales < ActiveRecord::Migration[5.2]
+  class User < ApplicationRecord
+    # Dummy class, to make migration possible across version changes
+  end
+
   disable_ddl_transaction!
 
   def up

db/migrate/20200620164023_add_fixed_lowercase_index_to_accounts.rb

@@ -1,16 +1,10 @@
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
 class AddFixedLowercaseIndexToAccounts < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
   disable_ddl_transaction!
 
-  class CorruptionError < StandardError
-    def cause
-      nil
-    end
-
-    def backtrace
-      []
-    end
-  end
-
   def up
     if index_name_exists?(:accounts, 'old_index_accounts_on_username_and_domain_lower') && index_name_exists?(:accounts, 'index_accounts_on_username_and_domain_lower')
       remove_index :accounts, name: 'index_accounts_on_username_and_domain_lower'
@@ -21,7 +15,8 @@ class AddFixedLowercaseIndexToAccounts < ActiveRecord::Migration[5.2]
     begin
       add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true, algorithm: :concurrently
     rescue ActiveRecord::RecordNotUnique
-      raise CorruptionError, 'Migration failed because of index corruption, see https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/#fixing'
+      remove_index :accounts, name: 'index_accounts_on_username_and_domain_lower'
+      raise CorruptionError
     end
 
     remove_index :accounts, name: 'old_index_accounts_on_username_and_domain_lower' if index_name_exists?(:accounts, 'old_index_accounts_on_username_and_domain_lower')

docker-compose.yml

@@ -43,7 +43,7 @@ services:
 
   web:
     build: .
-    image: tootsuite/mastodon
+    image: tootsuite/mastodon:v3.3.2
     restart: always
     env_file: .env.production
     command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
@@ -63,7 +63,7 @@ services:
 
   streaming:
     build: .
-    image: tootsuite/mastodon
+    image: tootsuite/mastodon:v3.3.2
     restart: always
     env_file: .env.production
     command: node ./streaming
@@ -80,7 +80,7 @@ services:
 
   sidekiq:
     build: .
-    image: tootsuite/mastodon
+    image: tootsuite/mastodon:v3.3.2
     restart: always
     env_file: .env.production
     command: bundle exec sidekiq

lib/cli.rb

@@ -94,17 +94,22 @@ module Mastodon
 
       exit(1) unless prompt.ask('Type in the domain of the server to confirm:', required: true) == Rails.configuration.x.local_domain
 
-      prompt.warn('This operation WILL NOT be reversible. It can also take a long time.')
-      prompt.warn('While the data won\'t be erased locally, the server will be in a BROKEN STATE afterwards.')
-      prompt.warn('A running Sidekiq process is required. Do not shut it down until queues clear.')
+      unless options[:dry_run]
+        prompt.warn('This operation WILL NOT be reversible. It can also take a long time.')
+        prompt.warn('While the data won\'t be erased locally, the server will be in a BROKEN STATE afterwards.')
+        prompt.warn('A running Sidekiq process is required. Do not shut it down until queues clear.')
 
-      exit(1) if prompt.no?('Are you sure you want to proceed?')
+        exit(1) if prompt.no?('Are you sure you want to proceed?')
+      end
 
       inboxes   = Account.inboxes
       processed = 0
       dry_run   = options[:dry_run] ? ' (DRY RUN)' : ''
 
+      Setting.registrations_mode = 'none' unless options[:dry_run]
+
       if inboxes.empty?
+        Account.local.without_suspended.in_batches.update_all(suspended_at: Time.now.utc, suspension_origin: :local) unless options[:dry_run]
         prompt.ok('It seems like your server has not federated with anything')
         prompt.ok('You can shut it down and delete it any time')
         return
@@ -112,9 +117,7 @@ module Mastodon
 
       prompt.warn('Do NOT interrupt this process...')
 
-      Setting.registrations_mode = 'none'
-
-      Account.local.without_suspended.find_each do |account|
+      delete_account = ->(account) do
         payload = ActiveModelSerializers::SerializableResource.new(
           account,
           serializer: ActivityPub::DeleteActorSerializer,
@@ -128,12 +131,15 @@ module Mastodon
             [json, account.id, inbox_url]
           end
 
-          account.suspend!
+          account.suspend!(block_email: false)
         end
 
         processed += 1
       end
 
+      Account.local.without_suspended.find_each { |account| delete_account.call(account) }
+      Account.local.suspended.joins(:deletion_request).find_each { |account| delete_account.call(account) }
+
       prompt.ok("Queued #{inboxes.size * processed} items into Sidekiq for #{processed} accounts#{dry_run}")
       prompt.ok('Wait until Sidekiq processes all items, then you can shut everything down and delete the data')
     rescue TTY::Reader::InputInterrupt

lib/mastodon/accounts_cli.rb

@@ -402,7 +402,7 @@ module Mastodon
         exit(1)
       end
 
-      parallelize_with_progress(target_account.followers.local) do |account|
+      processed, = parallelize_with_progress(target_account.followers.local) do |account|
         UnfollowService.new.call(account, target_account)
       end
 

lib/mastodon/emoji_cli.rb

@@ -43,8 +43,13 @@ module Mastodon
         tar.each do |entry|
           next unless entry.file? && entry.full_name.end_with?('.png')
 
-          shortcode    = [options[:prefix], File.basename(entry.full_name, '.*'), options[:suffix]].compact.join
-          custom_emoji = CustomEmoji.local.find_by(shortcode: shortcode)
+          filename = File.basename(entry.full_name, '.*')
+
+          # Skip macOS shadow files
+          next if filename.start_with?('._')
+
+          shortcode    = [options[:prefix], filename, options[:suffix]].compact.join
+          custom_emoji = CustomEmoji.local.find_by("LOWER(shortcode) = ?", shortcode.downcase)
 
           if custom_emoji && !options[:overwrite]
             skipped += 1

lib/mastodon/maintenance_cli.rb

@@ -14,7 +14,7 @@ module Mastodon
     end
 
     MIN_SUPPORTED_VERSION = 2019_10_01_213028
-    MAX_SUPPORTED_VERSION = 2020_12_18_054746
+    MAX_SUPPORTED_VERSION = 2021_03_08_133107
 
     # Stubs to enjoy ActiveRecord queries while not depending on a particular
     # version of the code/database
@@ -142,7 +142,6 @@ module Mastodon
       @prompt.warn 'Please make sure to stop Mastodon and have a backup.'
       exit(1) unless @prompt.yes?('Continue?')
 
-      deduplicate_accounts!
       deduplicate_users!
       deduplicate_account_domain_blocks!
       deduplicate_account_identity_proofs!
@@ -157,9 +156,11 @@ module Mastodon
       deduplicate_media_attachments!
       deduplicate_preview_cards!
       deduplicate_statuses!
+      deduplicate_accounts!
       deduplicate_tags!
       deduplicate_webauthn_credentials!
 
+      Scenic.database.refresh_materialized_view('instances', concurrently: true, cascade: false) if ActiveRecord::Migrator.current_version >= 2020_12_06_004238
       Rails.cache.clear
 
       @prompt.say 'Finished!'
@@ -188,6 +189,11 @@ module Mastodon
       else
         ActiveRecord::Base.connection.add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true
       end
+
+      @prompt.say 'Reindexing textual indexes on accounts…'
+      ActiveRecord::Base.connection.execute('REINDEX INDEX search_index;')
+      ActiveRecord::Base.connection.execute('REINDEX INDEX index_accounts_on_uri;')
+      ActiveRecord::Base.connection.execute('REINDEX INDEX index_accounts_on_url;')
     end
 
     def deduplicate_users!

lib/mastodon/migration_helpers.rb

@@ -41,6 +41,20 @@
 
 module Mastodon
   module MigrationHelpers
+    class CorruptionError < StandardError
+      def initialize(message = nil)
+        super(message.presence || 'Migration failed because of index corruption, see https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/#fixing')
+      end
+
+      def cause
+        nil
+      end
+
+      def backtrace
+        []
+      end
+    end
+
     # Stub for Database.postgresql? from GitLab
     def self.postgresql?
       ActiveRecord::Base.configurations[Rails.env]['adapter'].casecmp('postgresql').zero?
@@ -315,7 +329,7 @@ module Mastodon
       table = Arel::Table.new(table_name)
 
       total = estimate_rows_in_table(table_name).to_i
-      if total == 0
+      if total < 1
         count_arel = table.project(Arel.star.count.as('count'))
         count_arel = yield table, count_arel if block_given?
 

lib/mastodon/version.rb

@@ -13,7 +13,7 @@ module Mastodon
     end
 
     def patch
-      0
+      3
     end
 
     def flags

lib/paperclip/attachment_extensions.rb

@@ -2,6 +2,10 @@
 
 module Paperclip
   module AttachmentExtensions
+    def meta
+      instance_read(:meta)
+    end
+
     # We overwrite this method to support delayed processing in
     # Sidekiq. Since we process the original file to reduce disk
     # usage, and we still want to generate thumbnails straight

lib/paperclip/gif_transcoder.rb

@@ -100,7 +100,8 @@ end
 
 module Paperclip
   # This transcoder is only to be used for the MediaAttachment model
-  # to convert animated gifs to webm
+  # to convert animated GIFs to videos
+
   class GifTranscoder < Paperclip::Processor
     def make
       return File.open(@file.path) unless needs_convert?

lib/paperclip/image_extractor.rb

@@ -31,21 +31,17 @@ module Paperclip
     private
 
     def extract_image_from_file!
-      ::Av.logger = Paperclip.logger
-
-      cli = ::Av.cli
       dst = Tempfile.new([File.basename(@file.path, '.*'), '.png'])
       dst.binmode
 
-      cli.add_source(@file.path)
-      cli.add_destination(dst.path)
-      cli.add_output_param loglevel: 'fatal'
-
       begin
-        cli.run
-      rescue Cocaine::ExitStatusError, ::Av::CommandError
+        command = Terrapin::CommandLine.new('ffmpeg', '-i :source -loglevel :loglevel -y :destination', logger: Paperclip.logger)
+        command.run(source: @file.path, destination: dst.path, loglevel: 'fatal')
+      rescue Terrapin::ExitStatusError
         dst.close(true)
         return nil
+      rescue Terrapin::CommandNotFoundError
+        raise Paperclip::Errors::CommandNotFoundError, 'Could not run the `ffmpeg` command. Please install ffmpeg.'
       end
 
       dst

lib/paperclip/response_with_limit_adapter.rb

@@ -17,9 +17,9 @@ module Paperclip
 
     def cache_current_values
       @original_filename = filename_from_content_disposition.presence || filename_from_path.presence || 'data'
-      @size = @target.response.content_length
       @tempfile = copy_to_tempfile(@target)
       @content_type = ContentTypeDetector.new(@tempfile.path).detect
+      @size = File.size(@tempfile)
     end
 
     def copy_to_tempfile(source)

lib/paperclip/transcoder.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Paperclip
+  # This transcoder is only to be used for the MediaAttachment model
+  # to check when uploaded videos are actually gifv's
+  class Transcoder < Paperclip::Processor
+    def initialize(file, options = {}, attachment = nil)
+      super
+
+      @current_format      = File.extname(@file.path)
+      @basename            = File.basename(@file.path, @current_format)
+      @format              = options[:format]
+      @time                = options[:time] || 3
+      @passthrough_options = options[:passthrough_options]
+      @convert_options     = options[:convert_options].dup
+    end
+
+    def make
+      metadata = VideoMetadataExtractor.new(@file.path)
+
+      unless metadata.valid?
+        log("Unsupported file #{@file.path}")
+        return File.open(@file.path)
+      end
+
+      update_attachment_type(metadata)
+      update_options_from_metadata(metadata)
+
+      destination = Tempfile.new([@basename, @format ? ".#{@format}" : ''])
+      destination.binmode
+
+      @output_options = @convert_options[:output]&.dup || {}
+      @input_options  = @convert_options[:input]&.dup  || {}
+
+      case @format.to_s
+      when /jpg$/, /jpeg$/, /png$/, /gif$/
+        @input_options['ss'] = @time
+
+        @output_options['f']       = 'image2'
+        @output_options['vframes'] = 1
+      when 'mp4'
+        @output_options['acodec'] = 'aac'
+        @output_options['strict'] = 'experimental'
+      end
+
+      command_arguments, interpolations = prepare_command(destination)
+
+      begin
+        command = Terrapin::CommandLine.new('ffmpeg', command_arguments.join(' '), logger: Paperclip.logger)
+        command.run(interpolations)
+      rescue Terrapin::ExitStatusError => e
+        raise Paperclip::Error, "Error while transcoding #{@basename}: #{e}"
+      rescue Terrapin::CommandNotFoundError
+        raise Paperclip::Errors::CommandNotFoundError, 'Could not run the `ffmpeg` command. Please install ffmpeg.'
+      end
+
+      destination
+    end
+
+    private
+
+    def prepare_command(destination)
+      command_arguments  = ['-nostdin']
+      interpolations     = {}
+      interpolation_keys = 0
+
+      @input_options.each_pair do |key, value|
+        interpolation_key = interpolation_keys
+        command_arguments << "-#{key} :#{interpolation_key}"
+        interpolations[interpolation_key] = value
+        interpolation_keys += 1
+      end
+
+      command_arguments << '-i :source'
+      interpolations[:source] = @file.path
+
+      @output_options.each_pair do |key, value|
+        interpolation_key = interpolation_keys
+        command_arguments << "-#{key} :#{interpolation_key}"
+        interpolations[interpolation_key] = value
+        interpolation_keys += 1
+      end
+
+      command_arguments << '-y :destination'
+      interpolations[:destination] = destination.path
+
+      [command_arguments, interpolations]
+    end
+
+    def update_options_from_metadata(metadata)
+      return unless @passthrough_options && @passthrough_options[:video_codecs].include?(metadata.video_codec) && @passthrough_options[:audio_codecs].include?(metadata.audio_codec) && @passthrough_options[:colorspaces].include?(metadata.colorspace)
+
+      @format          = @passthrough_options[:options][:format] || @format
+      @time            = @passthrough_options[:options][:time]   || @time
+      @convert_options = @passthrough_options[:options][:convert_options].dup
+    end
+
+    def update_attachment_type(metadata)
+      @attachment.instance.type = MediaAttachment.types[:gifv] unless metadata.audio_codec
+    end
+  end
+end