From a6e9c41ed449021e5b84c7d0f48689336e4e7928 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 28 Jan 2022 20:38:38 +0100 Subject: [PATCH 01/70] Bump dependencies so that 3.3.x can be installed on current systems New system requirement: shared-mime-info --- Gemfile.lock | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index c4c8d9904..3df8f4bc5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -346,9 +346,11 @@ GEM mime-types (3.3.1) mime-types-data (~> 3.2015) mime-types-data (3.2020.0512) - mimemagic (0.3.5) + mimemagic (0.3.10) + nokogiri (~> 1) + rake mini_mime (1.0.2) - mini_portile2 (2.4.0) + mini_portile2 (2.7.1) minitest (5.14.2) msgpack (1.3.3) multi_json (1.15.0) @@ -358,9 +360,10 @@ GEM net-ssh (>= 2.6.5, < 7.0.0) net-ssh (6.1.0) nio4r (2.5.4) - nokogiri (1.10.10) - mini_portile2 (~> 2.4.0) - nokogumbo (2.0.2) + nokogiri (1.13.1) + mini_portile2 (~> 2.7.0) + racc (~> 1.4) + nokogumbo (2.0.5) nokogiri (~> 1.8, >= 1.8.4) nsa (0.2.7) activesupport (>= 4.2, < 6) @@ -432,6 +435,7 @@ GEM pundit (2.1.0) activesupport (>= 3.0.0) raabro (1.3.3) + racc (1.6.0) rack (2.2.3) rack-attack (6.3.1) rack (>= 1.0, < 3) From 64d9b84f1d12c180d4c379745e4f6b83687ba32e Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 5 May 2021 23:46:59 +0200 Subject: [PATCH 02/70] Fix media redownload worker retrying on unexpected response codes (#16111) --- app/lib/exceptions.rb | 4 ++++ app/workers/redownload_media_worker.rb | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/app/lib/exceptions.rb b/app/lib/exceptions.rb index 7c8e77871..eb472abaa 100644 --- a/app/lib/exceptions.rb +++ b/app/lib/exceptions.rb @@ -12,7 +12,11 @@ module Mastodon class RateLimitExceededError < Error; end class UnexpectedResponseError < Error + attr_reader :response + def initialize(response = nil) + @response = response + if response.respond_to? :uri super("#{response.uri} returned code #{response.code}") else diff --git a/app/workers/redownload_media_worker.rb b/app/workers/redownload_media_worker.rb index 0638cd0f0..343caa32c 100644 --- a/app/workers/redownload_media_worker.rb +++ b/app/workers/redownload_media_worker.rb @@ -3,6 +3,7 @@ class RedownloadMediaWorker include Sidekiq::Worker include ExponentialBackoff + include JsonLdHelper sidekiq_options queue: 'pull', retry: 3 @@ -15,6 +16,14 @@ class RedownloadMediaWorker media_attachment.download_thumbnail! media_attachment.save rescue ActiveRecord::RecordNotFound - true + # Do nothing + rescue Mastodon::UnexpectedResponseError => e + response = e.response + + if response_error_unsalvageable?(response) + # Give up + else + raise e + end end end From 02d195809bef48eeb017fecb45e1b33afbd9c35b Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 26 Apr 2021 18:56:45 +0200 Subject: [PATCH 03/70] Fix thread resolve worker retrying when status no longer exists (#16109) --- app/workers/thread_resolve_worker.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/workers/thread_resolve_worker.rb b/app/workers/thread_resolve_worker.rb index 8bba9ca75..1b77dfdd9 100644 --- a/app/workers/thread_resolve_worker.rb +++ b/app/workers/thread_resolve_worker.rb @@ -14,5 +14,7 @@ class ThreadResolveWorker child_status.thread = parent_status child_status.save! + rescue ActiveRecord::RecordNotFound + true end end From 47cab0500385b8b446a166b67d1b0f7243a8bc60 Mon Sep 17 00:00:00 2001 From: abcang Date: Mon, 1 Feb 2021 05:24:17 +0900 Subject: [PATCH 04/70] Fix N+1 query when rendering with StatusSerializer (#15641) --- app/models/status.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/status.rb b/app/models/status.rb index b426f9d5b..4353d2710 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -114,7 +114,7 @@ class Status < ApplicationRecord :tags, :preview_cards, :preloadable_poll, - account: :account_stat, + account: [:account_stat, :user], active_mentions: { account: :account_stat }, reblog: [ :application, @@ -124,7 +124,7 @@ class Status < ApplicationRecord :conversation, :status_stat, :preloadable_poll, - account: :account_stat, + account: [:account_stat, :user], active_mentions: { account: :account_stat }, ], thread: { account: :account_stat } @@ -301,7 +301,7 @@ class Status < ApplicationRecord return if account_ids.empty? - accounts = Account.where(id: account_ids).includes(:account_stat).each_with_object({}) { |a, h| h[a.id] = a } + accounts = Account.where(id: account_ids).includes(:account_stat, :user).index_by(&:id) cached_items.each do |item| item.account = accounts[item.account_id] From 4ea2c1da952d68fe8f77e59a9046176a115f5865 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 17 Apr 2021 14:55:46 +0200 Subject: [PATCH 05/70] Fix remote reporters not receiving suspend/unsuspend activities (#16050) --- app/lib/account_reach_finder.rb | 25 +++++++++++++++++++++++ app/services/suspend_account_service.rb | 12 ++++++++++- app/services/unsuspend_account_service.rb | 15 ++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 app/lib/account_reach_finder.rb diff --git a/app/lib/account_reach_finder.rb b/app/lib/account_reach_finder.rb new file mode 100644 index 000000000..706ce8c1f --- /dev/null +++ b/app/lib/account_reach_finder.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class AccountReachFinder + def initialize(account) + @account = account + end + + def inboxes + (followers_inboxes + reporters_inboxes + relay_inboxes).uniq + end + + private + + def followers_inboxes + @account.followers.inboxes + end + + def reporters_inboxes + Account.where(id: @account.targeted_reports.select(:account_id)).inboxes + end + + def relay_inboxes + Relay.enabled.pluck(:inbox_url) + end +end diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb index 9f4da91d4..b8dc8d5e0 100644 --- a/app/services/suspend_account_service.rb +++ b/app/services/suspend_account_service.rb @@ -42,7 +42,13 @@ class SuspendAccountService < BaseService end def distribute_update_actor! - ActivityPub::UpdateDistributionWorker.perform_async(@account.id) if @account.local? + return unless @account.local? + + account_reach_finder = AccountReachFinder.new(@account) + + ActivityPub::DeliveryWorker.push_bulk(account_reach_finder.inboxes) do |inbox_url| + [signed_activity_json, @account.id, inbox_url] + end end def unmerge_from_home_timelines! @@ -90,4 +96,8 @@ class SuspendAccountService < BaseService end end end + + def signed_activity_json + @signed_activity_json ||= Oj.dump(serialize_payload(@account, ActivityPub::UpdateSerializer, signer: @account)) + end end diff --git a/app/services/unsuspend_account_service.rb b/app/services/unsuspend_account_service.rb index ce9ee48ed..949c670aa 100644 --- a/app/services/unsuspend_account_service.rb +++ b/app/services/unsuspend_account_service.rb @@ -12,6 +12,7 @@ class UnsuspendAccountService < BaseService merge_into_home_timelines! merge_into_list_timelines! publish_media_attachments! + distribute_update_actor! end private @@ -36,6 +37,16 @@ class UnsuspendAccountService < BaseService # @account would now be nil. end + def distribute_update_actor! + return unless @account.local? + + account_reach_finder = AccountReachFinder.new(@account) + + ActivityPub::DeliveryWorker.push_bulk(account_reach_finder.inboxes) do |inbox_url| + [signed_activity_json, @account.id, inbox_url] + end + end + def merge_into_home_timelines! @account.followers_for_local_distribution.find_each do |follower| FeedManager.instance.merge_into_home(@account, follower) @@ -81,4 +92,8 @@ class UnsuspendAccountService < BaseService end end end + + def signed_activity_json + @signed_activity_json ||= Oj.dump(serialize_payload(@account, ActivityPub::UpdateSerializer, signer: @account)) + end end From 4063bbe04e0719e10a3b149a63a96fc558f0a20f Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 24 Mar 2021 10:19:40 +0100 Subject: [PATCH 06/70] Fix Mastodon not understanding as:Public and Public (#15948) Fixes #5551 --- app/lib/activitypub/activity/announce.rb | 4 +- app/lib/activitypub/activity/create.rb | 6 +- app/lib/activitypub/tag_manager.rb | 4 ++ spec/lib/activitypub/activity/create_spec.rb | 76 +++++++++++++++++++- 4 files changed, 83 insertions(+), 7 deletions(-) diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index 349e8f77e..ae8b2db75 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -43,9 +43,9 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity end def visibility_from_audience - if audience_to.include?(ActivityPub::TagManager::COLLECTIONS[:public]) + if audience_to.any? { |to| ActivityPub::TagManager.instance.public_collection?(to) } :public - elsif audience_cc.include?(ActivityPub::TagManager::COLLECTIONS[:public]) + elsif audience_cc.any? { |cc| ActivityPub::TagManager.instance.public_collection?(cc) } :unlisted elsif audience_to.include?(@account.followers_url) :private diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 612744676..0fa306cdd 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -123,7 +123,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def process_audience (audience_to + audience_cc).uniq.each do |audience| - next if audience == ActivityPub::TagManager::COLLECTIONS[:public] + next if ActivityPub::TagManager.instance.public_collection?(audience) # Unlike with tags, there is no point in resolving accounts we don't already # know here, because silent mentions would only be used for local access @@ -356,9 +356,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def visibility_from_audience - if audience_to.include?(ActivityPub::TagManager::COLLECTIONS[:public]) + if audience_to.any? { |to| ActivityPub::TagManager.instance.public_collection?(to) } :public - elsif audience_cc.include?(ActivityPub::TagManager::COLLECTIONS[:public]) + elsif audience_cc.any? { |cc| ActivityPub::TagManager.instance.public_collection?(cc) } :unlisted elsif audience_to.include?(@account.followers_url) :private diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb index 3f2ae1106..f6b5e10d3 100644 --- a/app/lib/activitypub/tag_manager.rb +++ b/app/lib/activitypub/tag_manager.rb @@ -12,6 +12,10 @@ class ActivityPub::TagManager public: 'https://www.w3.org/ns/activitystreams#Public', }.freeze + def public_collection?(uri) + uri == COLLECTIONS[:public] || uri == 'as:Public' || uri == 'Public' + end + def url_for(target) return target.url if target.respond_to?(:local?) && !target.local? diff --git a/spec/lib/activitypub/activity/create_spec.rb b/spec/lib/activitypub/activity/create_spec.rb index d2e9fe33c..2703c18f3 100644 --- a/spec/lib/activitypub/activity/create_spec.rb +++ b/spec/lib/activitypub/activity/create_spec.rb @@ -67,7 +67,7 @@ RSpec.describe ActivityPub::Activity::Create do end end - context 'public' do + context 'public with explicit public address' do let(:object_json) do { id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, @@ -85,7 +85,43 @@ RSpec.describe ActivityPub::Activity::Create do end end - context 'unlisted' do + context 'public with as:Public' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + to: 'as:Public', + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.visibility).to eq 'public' + end + end + + context 'public with Public' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + to: 'Public', + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.visibility).to eq 'public' + end + end + + context 'unlisted with explicit public address' do let(:object_json) do { id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, @@ -103,6 +139,42 @@ RSpec.describe ActivityPub::Activity::Create do end end + context 'unlisted with as:Public' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + cc: 'as:Public', + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.visibility).to eq 'unlisted' + end + end + + context 'unlisted with Public' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + cc: 'Public', + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.visibility).to eq 'unlisted' + end + end + context 'private' do let(:object_json) do { From 6386421d1a095bc60a6f80a6aa21309c288da284 Mon Sep 17 00:00:00 2001 From: ThibG Date: Wed, 30 Dec 2020 23:19:18 +0100 Subject: [PATCH 07/70] Fix profile update not being sent on profile/header picture deletion (#15461) Fixes #15460 Co-authored-by: Claire --- app/controllers/settings/pictures_controller.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/controllers/settings/pictures_controller.rb b/app/controllers/settings/pictures_controller.rb index 28df65f8f..58a432530 100644 --- a/app/controllers/settings/pictures_controller.rb +++ b/app/controllers/settings/pictures_controller.rb @@ -7,8 +7,12 @@ module Settings def destroy if valid_picture? - msg = I18n.t('generic.changes_saved_msg') if UpdateAccountService.new.call(@account, { @picture => nil, "#{@picture}_remote_url" => '' }) - redirect_to settings_profile_path, notice: msg, status: 303 + if UpdateAccountService.new.call(@account, { @picture => nil, "#{@picture}_remote_url" => '' }) + ActivityPub::UpdateDistributionWorker.perform_async(@account.id) + redirect_to settings_profile_path, notice: I18n.t('generic.changes_saved_msg'), status: 303 + else + redirect_to settings_profile_path + end else bad_request end From 13d1111a922f8341d2cfba823aec8ae4ce156dc1 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 21 Apr 2021 04:46:09 +0200 Subject: [PATCH 08/70] Fix processing of remote Delete activities (#16084) * Add tests * Ensure deleted statuses are marked as such * Save some redis memory by not storing URIs in delete_upon_arrival values * Avoid possible race condition when processing incoming Deletes * Avoid potential duplicate Delete forwards * Lower lock durations to reduce issues in case of hard crash of the Rails process * Check for `lock.aquired?` and improve comment * Refactor RedisLock usage in app/lib/activitypub * Fix using incorrect or non-existent sender for relaying Deletes --- app/lib/activitypub/activity.rb | 14 +++- app/lib/activitypub/activity/announce.rb | 36 ++++------- app/lib/activitypub/activity/create.rb | 36 +++-------- app/lib/activitypub/activity/delete.rb | 68 +++++++++++--------- app/services/remove_status_service.rb | 2 + spec/lib/activitypub/activity/delete_spec.rb | 20 ++++++ 6 files changed, 94 insertions(+), 82 deletions(-) diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index 2b5d3ffc2..3baee4ca4 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -144,7 +144,7 @@ class ActivityPub::Activity end def delete_later!(uri) - redis.setex("delete_upon_arrival:#{@account.id}:#{uri}", 6.hours.seconds, uri) + redis.setex("delete_upon_arrival:#{@account.id}:#{uri}", 6.hours.seconds, true) end def status_from_object @@ -210,12 +210,22 @@ class ActivityPub::Activity end end - def lock_or_return(key, expire_after = 7.days.seconds) + def lock_or_return(key, expire_after = 2.hours.seconds) yield if redis.set(key, true, nx: true, ex: expire_after) ensure redis.del(key) end + def lock_or_fail(key) + RedisLock.acquire({ redis: Redis.current, key: key }) do |lock| + if lock.acquired? + yield + else + raise Mastodon::RaceConditionError + end + end + end + def fetch? !@options[:delivery] end diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index ae8b2db75..a1081522e 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -4,29 +4,25 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity def perform return reject_payload! if delete_arrived_first?(@json['id']) || !related_to_local_activity? - RedisLock.acquire(lock_options) do |lock| - if lock.acquired? - original_status = status_from_object + lock_or_fail("announce:#{@object['id']}") do + original_status = status_from_object - return reject_payload! if original_status.nil? || !announceable?(original_status) + return reject_payload! if original_status.nil? || !announceable?(original_status) - @status = Status.find_by(account: @account, reblog: original_status) + @status = Status.find_by(account: @account, reblog: original_status) - return @status unless @status.nil? + return @status unless @status.nil? - @status = Status.create!( - account: @account, - reblog: original_status, - uri: @json['id'], - created_at: @json['published'], - override_timestamps: @options[:override_timestamps], - visibility: visibility_from_audience - ) + @status = Status.create!( + account: @account, + reblog: original_status, + uri: @json['id'], + created_at: @json['published'], + override_timestamps: @options[:override_timestamps], + visibility: visibility_from_audience + ) - distribute(@status) - else - raise Mastodon::RaceConditionError - end + distribute(@status) end @status @@ -69,8 +65,4 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity def reblog_of_local_status? status_from_uri(object_uri)&.account&.local? end - - def lock_options - { redis: Redis.current, key: "announce:#{@object['id']}" } - end end diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 0fa306cdd..0c0bfcba0 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -45,19 +45,15 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def create_status return reject_payload! if unsupported_object_type? || invalid_origin?(object_uri) || tombstone_exists? || !related_to_local_activity? - RedisLock.acquire(lock_options) do |lock| - if lock.acquired? - return if delete_arrived_first?(object_uri) || poll_vote? # rubocop:disable Lint/NonLocalExitFromIterator + lock_or_fail("create:#{object_uri}") do + return if delete_arrived_first?(object_uri) || poll_vote? # rubocop:disable Lint/NonLocalExitFromIterator - @status = find_existing_status + @status = find_existing_status - if @status.nil? - process_status - elsif @options[:delivered_to_account_id].present? - postprocess_audience_and_deliver - end - else - raise Mastodon::RaceConditionError + if @status.nil? + process_status + elsif @options[:delivered_to_account_id].present? + postprocess_audience_and_deliver end end @@ -314,13 +310,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity poll = replied_to_status.preloadable_poll already_voted = true - RedisLock.acquire(poll_lock_options) do |lock| - if lock.acquired? - already_voted = poll.votes.where(account: @account).exists? - poll.votes.create!(account: @account, choice: poll.options.index(@object['name']), uri: object_uri) - else - raise Mastodon::RaceConditionError - end + lock_or_fail("vote:#{replied_to_status.poll_id}:#{@account.id}") do + already_voted = poll.votes.where(account: @account).exists? + poll.votes.create!(account: @account, choice: poll.options.index(@object['name']), uri: object_uri) end increment_voters_count! unless already_voted @@ -513,12 +505,4 @@ class ActivityPub::Activity::Create < ActivityPub::Activity poll.reload retry end - - def lock_options - { redis: Redis.current, key: "create:#{object_uri}" } - end - - def poll_lock_options - { redis: Redis.current, key: "vote:#{replied_to_status.poll_id}:#{@account.id}" } - end end diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb index 2e5293b83..801647cf7 100644 --- a/app/lib/activitypub/activity/delete.rb +++ b/app/lib/activitypub/activity/delete.rb @@ -20,33 +20,35 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity def delete_note return if object_uri.nil? - unless invalid_origin?(object_uri) - RedisLock.acquire(lock_options) { |_lock| delete_later!(object_uri) } - Tombstone.find_or_create_by(uri: object_uri, account: @account) + lock_or_return("delete_status_in_progress:#{object_uri}", 5.minutes.seconds) do + unless invalid_origin?(object_uri) + # This lock ensures a concurrent `ActivityPub::Activity::Create` either + # does not create a status at all, or has finished saving it to the + # database before we try to load it. + # Without the lock, `delete_later!` could be called after `delete_arrived_first?` + # and `Status.find` before `Status.create!` + lock_or_fail("create:#{object_uri}") { delete_later!(object_uri) } + + Tombstone.find_or_create_by(uri: object_uri, account: @account) + end + + @status = Status.find_by(uri: object_uri, account: @account) + @status ||= Status.find_by(uri: @object['atomUri'], account: @account) if @object.is_a?(Hash) && @object['atomUri'].present? + + return if @status.nil? + + forward! if @json['signature'].present? && @status.distributable? + delete_now! end - - @status = Status.find_by(uri: object_uri, account: @account) - @status ||= Status.find_by(uri: @object['atomUri'], account: @account) if @object.is_a?(Hash) && @object['atomUri'].present? - - return if @status.nil? - - if @status.distributable? - forward_for_reply - forward_for_reblogs - end - - delete_now! end - def forward_for_reblogs - return if @json['signature'].blank? + def rebloggers_ids + return @rebloggers_ids if defined?(@rebloggers_ids) + @rebloggers_ids = @status.reblogs.includes(:account).references(:account).merge(Account.local).pluck(:account_id) + end - rebloggers_ids = @status.reblogs.includes(:account).references(:account).merge(Account.local).pluck(:account_id) - inboxes = Account.where(id: ::Follow.where(target_account_id: rebloggers_ids).select(:account_id)).inboxes - [@account.preferred_inbox_url] - - ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes) do |inbox_url| - [payload, rebloggers_ids.first, inbox_url] - end + def inboxes_for_reblogs + Account.where(id: ::Follow.where(target_account_id: rebloggers_ids).select(:account_id)).inboxes end def replied_to_status @@ -58,13 +60,19 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity !replied_to_status.nil? && replied_to_status.account.local? end - def forward_for_reply - return unless @json['signature'].present? && reply_to_local? + def inboxes_for_reply + replied_to_status.account.followers.inboxes + end - inboxes = replied_to_status.account.followers.inboxes - [@account.preferred_inbox_url] + def forward! + inboxes = inboxes_for_reblogs + inboxes += inboxes_for_reply if reply_to_local? + inboxes -= [@account.preferred_inbox_url] - ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes) do |inbox_url| - [payload, replied_to_status.account_id, inbox_url] + sender_id = reply_to_local? ? replied_to_status.account_id : rebloggers_ids.first + + ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes.uniq) do |inbox_url| + [payload, sender_id, inbox_url] end end @@ -75,8 +83,4 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity def payload @payload ||= Oj.dump(@json) end - - def lock_options - { redis: Redis.current, key: "create:#{object_uri}" } - end end diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index d6043fb5d..52d3f108c 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -16,6 +16,8 @@ class RemoveStatusService < BaseService @account = status.account @options = options + @status.discard + RedisLock.acquire(lock_options) do |lock| if lock.acquired? remove_from_self if @account.local? diff --git a/spec/lib/activitypub/activity/delete_spec.rb b/spec/lib/activitypub/activity/delete_spec.rb index 37b93ecf7..9dfb8a61b 100644 --- a/spec/lib/activitypub/activity/delete_spec.rb +++ b/spec/lib/activitypub/activity/delete_spec.rb @@ -49,4 +49,24 @@ RSpec.describe ActivityPub::Activity::Delete do end end end + + context 'when the status has been reported' do + describe '#perform' do + subject { described_class.new(json, sender) } + let!(:reporter) { Fabricate(:account) } + + before do + reporter.reports.create!(target_account: status.account, status_ids: [status.id], forwarded: false) + subject.perform + end + + it 'marks the status as deleted' do + expect(Status.find_by(id: status.id)).to be_nil + end + + it 'actually keeps a copy for inspection' do + expect(Status.with_discarded.find_by(id: status.id)).to_not be_nil + end + end + end end From a2dc4e583bf57f09ad36fef8248e2106b25470c1 Mon Sep 17 00:00:00 2001 From: ThibG Date: Tue, 12 Jan 2021 09:25:01 +0100 Subject: [PATCH 09/70] Fix processing of incoming Block activities (#15546) Unlike locally-issued blocks, they weren't clearing follow relationships in both directions, follow requests or notifications. Co-authored-by: Claire --- app/lib/activitypub/activity/block.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/lib/activitypub/activity/block.rb b/app/lib/activitypub/activity/block.rb index 90477bf33..92a0f813f 100644 --- a/app/lib/activitypub/activity/block.rb +++ b/app/lib/activitypub/activity/block.rb @@ -11,8 +11,13 @@ class ActivityPub::Activity::Block < ActivityPub::Activity return end + UnfollowService.new.call(@account, target_account) if @account.following?(target_account) UnfollowService.new.call(target_account, @account) if target_account.following?(@account) + RejectFollowService.new.call(target_account, @account) if target_account.requested?(@account) - @account.block!(target_account, uri: @json['id']) unless delete_arrived_first?(@json['id']) + unless delete_arrived_first?(@json['id']) + BlockWorker.perform_async(@account.id, target_account.id) + @account.block!(target_account, uri: @json['id']) + end end end From 685cde55cbb3a5af3163d2a735f8a020e25b5b5c Mon Sep 17 00:00:00 2001 From: ThibG Date: Fri, 8 Jan 2021 12:26:29 +0100 Subject: [PATCH 10/70] Skip processing Update activities on unknown accounts (#15514) This also skips fetching the actor completely. This will be useful if we end up distributing Update activities linked to account suspensions more widely (they are currently only delivered to the suspended account's followers), as currently, instances not knowing about the suspended account would fetch it to then process the suspension. Co-authored-by: Claire --- app/controllers/activitypub/inboxes_controller.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/controllers/activitypub/inboxes_controller.rb b/app/controllers/activitypub/inboxes_controller.rb index d3044f180..92dcb5ac7 100644 --- a/app/controllers/activitypub/inboxes_controller.rb +++ b/app/controllers/activitypub/inboxes_controller.rb @@ -5,7 +5,7 @@ class ActivityPub::InboxesController < ActivityPub::BaseController include JsonLdHelper include AccountOwnedConcern - before_action :skip_unknown_actor_delete + before_action :skip_unknown_actor_activity before_action :require_signature! skip_before_action :authenticate_user! @@ -18,13 +18,13 @@ class ActivityPub::InboxesController < ActivityPub::BaseController private - def skip_unknown_actor_delete - head 202 if unknown_deleted_account? + def skip_unknown_actor_activity + head 202 if unknown_affected_account? end - def unknown_deleted_account? + def unknown_affected_account? json = Oj.load(body, mode: :strict) - json.is_a?(Hash) && json['type'] == 'Delete' && json['actor'].present? && json['actor'] == value_or_id(json['object']) && !Account.where(uri: json['actor']).exists? + json.is_a?(Hash) && %w(Delete Update).include?(json['type']) && json['actor'].present? && json['actor'] == value_or_id(json['object']) && !Account.where(uri: json['actor']).exists? rescue Oj::ParseError false end From cc21670b3caebb875d6f32fecef9f93db4f3e036 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 11 Feb 2021 01:53:44 +0100 Subject: [PATCH 11/70] Fix URI of repeat follow requests not being recorded (#15662) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix URI of repeat follow requests not being recorded In case we receive a “repeat” or “duplicate” follow request, we automatically fast-forward the accept with the latest received Activity `id`, but we don't record it. In general, a “repeat” or “duplicate” follow request may happen if for some reason (e.g. inconsistent handling of Block or Undo Accept activities, an instance being brought back up from the dead, etc.) the local instance thought the remote actor were following them while the remote actor thought otherwise. In those cases, the remote instance does not know about the older Follow activity `id`, so keeping that record serves no purpose, but knowing the most recent one is useful if the remote implementation at some point refers to it by `id` without inlining it. * Add tests --- app/lib/activitypub/activity/follow.rb | 13 +- spec/lib/activitypub/activity/follow_spec.rb | 177 +++++++++++++++---- 2 files changed, 154 insertions(+), 36 deletions(-) diff --git a/app/lib/activitypub/activity/follow.rb b/app/lib/activitypub/activity/follow.rb index 0beec68ab..4efb84b8c 100644 --- a/app/lib/activitypub/activity/follow.rb +++ b/app/lib/activitypub/activity/follow.rb @@ -6,7 +6,14 @@ class ActivityPub::Activity::Follow < ActivityPub::Activity def perform target_account = account_from_uri(object_uri) - return if target_account.nil? || !target_account.local? || delete_arrived_first?(@json['id']) || @account.requested?(target_account) + return if target_account.nil? || !target_account.local? || delete_arrived_first?(@json['id']) + + # Update id of already-existing follow requests + existing_follow_request = ::FollowRequest.find_by(account: @account, target_account: target_account) + unless existing_follow_request.nil? + existing_follow_request.update!(uri: @json['id']) + return + end if target_account.blocking?(@account) || target_account.domain_blocking?(@account.domain) || target_account.moved? || target_account.instance_actor? reject_follow_request!(target_account) @@ -14,7 +21,9 @@ class ActivityPub::Activity::Follow < ActivityPub::Activity end # Fast-forward repeat follow requests - if @account.following?(target_account) + existing_follow = ::Follow.find_by(account: @account, target_account: target_account) + unless existing_follow.nil? + existing_follow.update!(uri: @json['id']) AuthorizeFollowService.new.call(@account, target_account, skip_follow_request: true, follow_request_uri: @json['id']) return end diff --git a/spec/lib/activitypub/activity/follow_spec.rb b/spec/lib/activitypub/activity/follow_spec.rb index 05112cc18..fd4ede82b 100644 --- a/spec/lib/activitypub/activity/follow_spec.rb +++ b/spec/lib/activitypub/activity/follow_spec.rb @@ -17,62 +17,171 @@ RSpec.describe ActivityPub::Activity::Follow do describe '#perform' do subject { described_class.new(json, sender) } - context 'unlocked account' do - before do - subject.perform + context 'with no prior follow' do + context 'unlocked account' do + before do + subject.perform + end + + it 'creates a follow from sender to recipient' do + expect(sender.following?(recipient)).to be true + expect(sender.active_relationships.find_by(target_account: recipient).uri).to eq 'foo' + end + + it 'does not create a follow request' do + expect(sender.requested?(recipient)).to be false + end end - it 'creates a follow from sender to recipient' do - expect(sender.following?(recipient)).to be true + context 'silenced account following an unlocked account' do + before do + sender.touch(:silenced_at) + subject.perform + end + + it 'does not create a follow from sender to recipient' do + expect(sender.following?(recipient)).to be false + end + + it 'creates a follow request' do + expect(sender.requested?(recipient)).to be true + expect(sender.follow_requests.find_by(target_account: recipient).uri).to eq 'foo' + end end - it 'does not create a follow request' do - expect(sender.requested?(recipient)).to be false + context 'unlocked account muting the sender' do + before do + recipient.mute!(sender) + subject.perform + end + + it 'creates a follow from sender to recipient' do + expect(sender.following?(recipient)).to be true + expect(sender.active_relationships.find_by(target_account: recipient).uri).to eq 'foo' + end + + it 'does not create a follow request' do + expect(sender.requested?(recipient)).to be false + end + end + + context 'locked account' do + before do + recipient.update(locked: true) + subject.perform + end + + it 'does not create a follow from sender to recipient' do + expect(sender.following?(recipient)).to be false + end + + it 'creates a follow request' do + expect(sender.requested?(recipient)).to be true + expect(sender.follow_requests.find_by(target_account: recipient).uri).to eq 'foo' + end end end - context 'silenced account following an unlocked account' do + context 'when a follow relationship already exists' do before do - sender.touch(:silenced_at) - subject.perform + sender.active_relationships.create!(target_account: recipient, uri: 'bar') end - it 'does not create a follow from sender to recipient' do - expect(sender.following?(recipient)).to be false + context 'unlocked account' do + before do + subject.perform + end + + it 'correctly sets the new URI' do + expect(sender.active_relationships.find_by(target_account: recipient).uri).to eq 'foo' + end + + it 'does not create a follow request' do + expect(sender.requested?(recipient)).to be false + end end - it 'creates a follow request' do - expect(sender.requested?(recipient)).to be true + context 'silenced account following an unlocked account' do + before do + sender.touch(:silenced_at) + subject.perform + end + + it 'correctly sets the new URI' do + expect(sender.active_relationships.find_by(target_account: recipient).uri).to eq 'foo' + end + + it 'does not create a follow request' do + expect(sender.requested?(recipient)).to be false + end + end + + context 'unlocked account muting the sender' do + before do + recipient.mute!(sender) + subject.perform + end + + it 'correctly sets the new URI' do + expect(sender.active_relationships.find_by(target_account: recipient).uri).to eq 'foo' + end + + it 'does not create a follow request' do + expect(sender.requested?(recipient)).to be false + end + end + + context 'locked account' do + before do + recipient.update(locked: true) + subject.perform + end + + it 'correctly sets the new URI' do + expect(sender.active_relationships.find_by(target_account: recipient).uri).to eq 'foo' + end + + it 'does not create a follow request' do + expect(sender.requested?(recipient)).to be false + end end end - context 'unlocked account muting the sender' do + context 'when a follow request already exists' do before do - recipient.mute!(sender) - subject.perform + sender.follow_requests.create!(target_account: recipient, uri: 'bar') end - it 'creates a follow from sender to recipient' do - expect(sender.following?(recipient)).to be true + context 'silenced account following an unlocked account' do + before do + sender.touch(:silenced_at) + subject.perform + end + + it 'does not create a follow from sender to recipient' do + expect(sender.following?(recipient)).to be false + end + + it 'correctly sets the new URI' do + expect(sender.requested?(recipient)).to be true + expect(sender.follow_requests.find_by(target_account: recipient).uri).to eq 'foo' + end end - it 'does not create a follow request' do - expect(sender.requested?(recipient)).to be false - end - end + context 'locked account' do + before do + recipient.update(locked: true) + subject.perform + end - context 'locked account' do - before do - recipient.update(locked: true) - subject.perform - end + it 'does not create a follow from sender to recipient' do + expect(sender.following?(recipient)).to be false + end - it 'does not create a follow from sender to recipient' do - expect(sender.following?(recipient)).to be false - end - - it 'creates a follow request' do - expect(sender.requested?(recipient)).to be true + it 'correctly sets the new URI' do + expect(sender.requested?(recipient)).to be true + expect(sender.follow_requests.find_by(target_account: recipient).uri).to eq 'foo' + end end end end From da14725a96619f1330e19478cc2393c2f690c2bf Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 26 Feb 2021 17:40:27 +0100 Subject: [PATCH 12/70] Fix crash on receiving requests with missing Digest header (#15782) * Fix crash on receiving requests with missing Digest header Return an error pointing out that Digest is missing, instead of crashing. Fixes #15743 * Fix from review feedback --- app/controllers/concerns/signature_verification.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/controllers/concerns/signature_verification.rb b/app/controllers/concerns/signature_verification.rb index fc3978fbb..4dd0cac55 100644 --- a/app/controllers/concerns/signature_verification.rb +++ b/app/controllers/concerns/signature_verification.rb @@ -133,6 +133,7 @@ module SignatureVerification def verify_body_digest! return unless signed_headers.include?('digest') + raise SignatureVerificationError, 'Digest header missing' unless request.headers.key?('Digest') digests = request.headers['Digest'].split(',').map { |digest| digest.split('=', 2) }.map { |key, value| [key.downcase, value] } sha256 = digests.assoc('sha-256') From 48b25e457d26b9af290b0d0a4aeb0c3d9cf5cd61 Mon Sep 17 00:00:00 2001 From: ThibG Date: Tue, 19 Jan 2021 06:47:36 +0100 Subject: [PATCH 13/70] Fix /activity endpoint not require signature in authorized fetch mode (#15592) Fixes #15589 Co-authored-by: Claire --- app/controllers/statuses_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/statuses_controller.rb b/app/controllers/statuses_controller.rb index 17ddd31fb..87612a296 100644 --- a/app/controllers/statuses_controller.rb +++ b/app/controllers/statuses_controller.rb @@ -8,7 +8,7 @@ class StatusesController < ApplicationController layout 'public' - before_action :require_signature!, only: :show, if: -> { request.format == :json && authorized_fetch_mode? } + before_action :require_signature!, only: [:show, :activity], if: -> { request.format == :json && authorized_fetch_mode? } before_action :set_status before_action :set_instance_presenter before_action :set_link_headers From b593a7da8c894c14fdc75a098aed5b32099fa7fa Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 5 May 2021 19:44:35 +0200 Subject: [PATCH 14/70] Fix database serialization failure returning HTTP 500 (#16101) Database serialization failure occurs when a read-replica is used and a query takes long enough that rows on the primary database become unavailable. It should return HTTP 503 as it is temporary. Re-order rescue definitions according to their status codes --- app/controllers/application_controller.rb | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 44616d6e5..d47bdf3ac 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -20,17 +20,16 @@ class ApplicationController < ActionController::Base helper_method :use_seamless_external_login? helper_method :whitelist_mode? - rescue_from ActionController::RoutingError, with: :not_found - rescue_from ActionController::InvalidAuthenticityToken, with: :unprocessable_entity - rescue_from ActionController::UnknownFormat, with: :not_acceptable - rescue_from ActionController::ParameterMissing, with: :bad_request - rescue_from Paperclip::AdapterRegistry::NoHandlerError, with: :bad_request - rescue_from ActiveRecord::RecordNotFound, with: :not_found + rescue_from ActionController::ParameterMissing, Paperclip::AdapterRegistry::NoHandlerError, with: :bad_request rescue_from Mastodon::NotPermittedError, with: :forbidden - rescue_from HTTP::Error, OpenSSL::SSL::SSLError, with: :internal_server_error - rescue_from Mastodon::RaceConditionError, Seahorse::Client::NetworkingError, Stoplight::Error::RedLight, with: :service_unavailable + rescue_from ActionController::RoutingError, ActiveRecord::RecordNotFound, with: :not_found + rescue_from ActionController::UnknownFormat, with: :not_acceptable + rescue_from ActionController::InvalidAuthenticityToken, with: :unprocessable_entity rescue_from Mastodon::RateLimitExceededError, with: :too_many_requests + rescue_from HTTP::Error, OpenSSL::SSL::SSLError, with: :internal_server_error + rescue_from Mastodon::RaceConditionError, Seahorse::Client::NetworkingError, Stoplight::Error::RedLight, ActiveRecord::SerializationFailure, with: :service_unavailable + before_action :store_current_location, except: :raise_not_found, unless: :devise_controller? before_action :require_functional!, if: :user_signed_in? From 4b9a0cfe5ed35604a7eca9f7a3d3956d1a796b6a Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 5 May 2021 19:44:01 +0200 Subject: [PATCH 15/70] Fix media processing getting stuck on too much stdin/stderr (#16136) * Fix media processing getting stuck on too much stdin/stderr See thoughtbot/terrapin#5 * Remove dependency on paperclip-av-transcoder gem * Remove dependency on streamio-ffmpeg gem * Disable stdin on ffmpeg process --- Gemfile | 2 - Gemfile.lock | 11 --- app/lib/video_metadata_extractor.rb | 54 +++++++++++++ app/models/media_attachment.rb | 4 +- config/application.rb | 4 +- lib/paperclip/attachment_extensions.rb | 4 + lib/paperclip/gif_transcoder.rb | 3 +- lib/paperclip/image_extractor.rb | 14 ++-- lib/paperclip/transcoder.rb | 102 +++++++++++++++++++++++++ lib/paperclip/transcoder_extensions.rb | 14 ---- lib/paperclip/video_transcoder.rb | 26 ------- lib/terrapin/multi_pipe_extensions.rb | 63 +++++++++++++++ 12 files changed, 234 insertions(+), 67 deletions(-) create mode 100644 app/lib/video_metadata_extractor.rb create mode 100644 lib/paperclip/transcoder.rb delete mode 100644 lib/paperclip/transcoder_extensions.rb delete mode 100644 lib/paperclip/video_transcoder.rb create mode 100644 lib/terrapin/multi_pipe_extensions.rb diff --git a/Gemfile b/Gemfile index 3aee5d7cc..5d5fa144b 100644 --- a/Gemfile +++ b/Gemfile @@ -21,8 +21,6 @@ gem 'aws-sdk-s3', '~> 1.85', require: false gem 'fog-core', '<= 2.1.0' gem 'fog-openstack', '~> 0.3', require: false gem 'paperclip', '~> 6.0' -gem 'paperclip-av-transcoder', '~> 0.6' -gem 'streamio-ffmpeg', '~> 3.0' gem 'blurhash', '~> 0.1' gem 'active_model_serializers', '~> 0.10' diff --git a/Gemfile.lock b/Gemfile.lock index 3df8f4bc5..9f4fe4e91 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -75,8 +75,6 @@ GEM ast (2.4.1) attr_encrypted (3.1.0) encryptor (~> 3.0.0) - av (0.9.0) - cocaine (~> 0.5.3) awrence (1.1.1) aws-eventstream (1.1.0) aws-partitions (1.397.0) @@ -151,8 +149,6 @@ GEM cld3 (3.3.0) ffi (>= 1.1.0, < 1.12.0) climate_control (0.2.0) - cocaine (0.5.8) - climate_control (>= 0.0.3, < 1.0) coderay (1.1.3) color_diff (0.1) concurrent-ruby (1.1.7) @@ -394,9 +390,6 @@ GEM mime-types mimemagic (~> 0.3.0) terrapin (~> 0.6.0) - paperclip-av-transcoder (0.6.4) - av (~> 0.9.0) - paperclip (>= 2.5.2) parallel (1.20.1) parallel_tests (3.4.0) parallel @@ -609,8 +602,6 @@ GEM stackprof (0.2.16) statsd-ruby (1.4.0) stoplight (2.2.1) - streamio-ffmpeg (3.0.2) - multi_json (~> 1.8) strong_migrations (0.7.2) activerecord (>= 5) temple (0.8.2) @@ -755,7 +746,6 @@ DEPENDENCIES omniauth-saml (~> 1.10) ox (~> 2.13) paperclip (~> 6.0) - paperclip-av-transcoder (~> 0.6) parallel (~> 1.20) parallel_tests (~> 3.4) parslet @@ -801,7 +791,6 @@ DEPENDENCIES sprockets-rails (~> 3.2) stackprof stoplight (~> 2.2.1) - streamio-ffmpeg (~> 3.0) strong_migrations (~> 0.7) thor (~> 1.0) tty-prompt (~> 0.22) diff --git a/app/lib/video_metadata_extractor.rb b/app/lib/video_metadata_extractor.rb new file mode 100644 index 000000000..03e40f923 --- /dev/null +++ b/app/lib/video_metadata_extractor.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +class VideoMetadataExtractor + attr_reader :duration, :bitrate, :video_codec, :audio_codec, + :colorspace, :width, :height, :frame_rate + + def initialize(path) + @path = path + @metadata = Oj.load(ffmpeg_command_output, mode: :strict, symbol_keys: true) + + parse_metadata + rescue Terrapin::ExitStatusError, Oj::ParseError + @invalid = true + rescue Terrapin::CommandNotFoundError + raise Paperclip::Errors::CommandNotFoundError, 'Could not run the `ffprobe` command. Please install ffmpeg.' + end + + def valid? + !@invalid + end + + private + + def ffmpeg_command_output + command = Terrapin::CommandLine.new('ffprobe', '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel') + command.run(path: @path, format: 'json', loglevel: 'fatal') + end + + def parse_metadata + if @metadata.key?(:format) + @duration = @metadata[:format][:duration].to_f + @bitrate = @metadata[:format][:bit_rate].to_i + end + + if @metadata.key?(:streams) + video_streams = @metadata[:streams].select { |stream| stream[:codec_type] == 'video' } + audio_streams = @metadata[:streams].select { |stream| stream[:codec_type] == 'audio' } + + if (video_stream = video_streams.first) + @video_codec = video_stream[:codec_name] + @colorspace = video_stream[:pix_fmt] + @width = video_stream[:width] + @height = video_stream[:height] + @frame_rate = video_stream[:avg_frame_rate] == '0/0' ? nil : Rational(video_stream[:avg_frame_rate]) + end + + if (audio_stream = audio_streams.first) + @audio_codec = audio_stream[:codec_name] + end + end + + @invalid = true if @metadata.key?(:error) + end +end diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 663bb0896..87e590b2d 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -287,7 +287,7 @@ class MediaAttachment < ApplicationRecord if instance.file_content_type == 'image/gif' [:gif_transcoder, :blurhash_transcoder] elsif VIDEO_MIME_TYPES.include?(instance.file_content_type) - [:video_transcoder, :blurhash_transcoder, :type_corrector] + [:transcoder, :blurhash_transcoder, :type_corrector] elsif AUDIO_MIME_TYPES.include?(instance.file_content_type) [:image_extractor, :transcoder, :type_corrector] else @@ -388,7 +388,7 @@ class MediaAttachment < ApplicationRecord # paths but ultimately the same file, so it makes sense to memoize the # result while disregarding the path def ffmpeg_data(path = nil) - @ffmpeg_data ||= FFMPEG::Movie.new(path) + @ffmpeg_data ||= VideoMetadataExtractor.new(path) end def enqueue_processing diff --git a/config/application.rb b/config/application.rb index af7735221..c9475ba04 100644 --- a/config/application.rb +++ b/config/application.rb @@ -11,12 +11,12 @@ require_relative '../lib/redis/namespace_extensions' require_relative '../lib/paperclip/url_generator_extensions' require_relative '../lib/paperclip/attachment_extensions' require_relative '../lib/paperclip/media_type_spoof_detector_extensions' -require_relative '../lib/paperclip/transcoder_extensions' require_relative '../lib/paperclip/lazy_thumbnail' require_relative '../lib/paperclip/gif_transcoder' -require_relative '../lib/paperclip/video_transcoder' +require_relative '../lib/paperclip/transcoder' require_relative '../lib/paperclip/type_corrector' require_relative '../lib/paperclip/response_with_limit_adapter' +require_relative '../lib/terrapin/multi_pipe_extensions' require_relative '../lib/mastodon/snowflake' require_relative '../lib/mastodon/version' require_relative '../lib/devise/two_factor_ldap_authenticatable' diff --git a/lib/paperclip/attachment_extensions.rb b/lib/paperclip/attachment_extensions.rb index 94f7769b6..2f2656616 100644 --- a/lib/paperclip/attachment_extensions.rb +++ b/lib/paperclip/attachment_extensions.rb @@ -2,6 +2,10 @@ module Paperclip module AttachmentExtensions + def meta + instance_read(:meta) + end + # We overwrite this method to support delayed processing in # Sidekiq. Since we process the original file to reduce disk # usage, and we still want to generate thumbnails straight diff --git a/lib/paperclip/gif_transcoder.rb b/lib/paperclip/gif_transcoder.rb index 9f3c8e8be..74aa1a0b2 100644 --- a/lib/paperclip/gif_transcoder.rb +++ b/lib/paperclip/gif_transcoder.rb @@ -100,7 +100,8 @@ end module Paperclip # This transcoder is only to be used for the MediaAttachment model - # to convert animated gifs to webm + # to convert animated GIFs to videos + class GifTranscoder < Paperclip::Processor def make return File.open(@file.path) unless needs_convert? diff --git a/lib/paperclip/image_extractor.rb b/lib/paperclip/image_extractor.rb index aab675a06..17fe4326f 100644 --- a/lib/paperclip/image_extractor.rb +++ b/lib/paperclip/image_extractor.rb @@ -31,21 +31,17 @@ module Paperclip private def extract_image_from_file! - ::Av.logger = Paperclip.logger - - cli = ::Av.cli dst = Tempfile.new([File.basename(@file.path, '.*'), '.png']) dst.binmode - cli.add_source(@file.path) - cli.add_destination(dst.path) - cli.add_output_param loglevel: 'fatal' - begin - cli.run - rescue Cocaine::ExitStatusError, ::Av::CommandError + command = Terrapin::CommandLine.new('ffmpeg', '-i :source -loglevel :loglevel -y :destination', logger: Paperclip.logger) + command.run(source: @file.path, destination: dst.path, loglevel: 'fatal') + rescue Terrapin::ExitStatusError dst.close(true) return nil + rescue Terrapin::CommandNotFoundError + raise Paperclip::Errors::CommandNotFoundError, 'Could not run the `ffmpeg` command. Please install ffmpeg.' end dst diff --git a/lib/paperclip/transcoder.rb b/lib/paperclip/transcoder.rb new file mode 100644 index 000000000..e99704086 --- /dev/null +++ b/lib/paperclip/transcoder.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +module Paperclip + # This transcoder is only to be used for the MediaAttachment model + # to check when uploaded videos are actually gifv's + class Transcoder < Paperclip::Processor + def initialize(file, options = {}, attachment = nil) + super + + @current_format = File.extname(@file.path) + @basename = File.basename(@file.path, @current_format) + @format = options[:format] + @time = options[:time] || 3 + @passthrough_options = options[:passthrough_options] + @convert_options = options[:convert_options].dup + end + + def make + metadata = VideoMetadataExtractor.new(@file.path) + + unless metadata.valid? + log("Unsupported file #{@file.path}") + return File.open(@file.path) + end + + update_attachment_type(metadata) + update_options_from_metadata(metadata) + + destination = Tempfile.new([@basename, @format ? ".#{@format}" : '']) + destination.binmode + + @output_options = @convert_options[:output]&.dup || {} + @input_options = @convert_options[:input]&.dup || {} + + case @format.to_s + when /jpg$/, /jpeg$/, /png$/, /gif$/ + @input_options['ss'] = @time + + @output_options['f'] = 'image2' + @output_options['vframes'] = 1 + when 'mp4' + @output_options['acodec'] = 'aac' + @output_options['strict'] = 'experimental' + end + + command_arguments, interpolations = prepare_command(destination) + + begin + command = Terrapin::CommandLine.new('ffmpeg', command_arguments.join(' '), logger: Paperclip.logger) + command.run(interpolations) + rescue Terrapin::ExitStatusError => e + raise Paperclip::Error, "Error while transcoding #{@basename}: #{e}" + rescue Terrapin::CommandNotFoundError + raise Paperclip::Errors::CommandNotFoundError, 'Could not run the `ffmpeg` command. Please install ffmpeg.' + end + + destination + end + + private + + def prepare_command(destination) + command_arguments = ['-nostdin'] + interpolations = {} + interpolation_keys = 0 + + @input_options.each_pair do |key, value| + interpolation_key = interpolation_keys + command_arguments << "-#{key} :#{interpolation_key}" + interpolations[interpolation_key] = value + interpolation_keys += 1 + end + + command_arguments << '-i :source' + interpolations[:source] = @file.path + + @output_options.each_pair do |key, value| + interpolation_key = interpolation_keys + command_arguments << "-#{key} :#{interpolation_key}" + interpolations[interpolation_key] = value + interpolation_keys += 1 + end + + command_arguments << '-y :destination' + interpolations[:destination] = destination.path + + [command_arguments, interpolations] + end + + def update_options_from_metadata(metadata) + return unless @passthrough_options && @passthrough_options[:video_codecs].include?(metadata.video_codec) && @passthrough_options[:audio_codecs].include?(metadata.audio_codec) && @passthrough_options[:colorspaces].include?(metadata.colorspace) + + @format = @passthrough_options[:options][:format] || @format + @time = @passthrough_options[:options][:time] || @time + @convert_options = @passthrough_options[:options][:convert_options].dup + end + + def update_attachment_type(metadata) + @attachment.instance.type = MediaAttachment.types[:gifv] unless metadata.audio_codec + end + end +end diff --git a/lib/paperclip/transcoder_extensions.rb b/lib/paperclip/transcoder_extensions.rb deleted file mode 100644 index c0b2447f3..000000000 --- a/lib/paperclip/transcoder_extensions.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Paperclip - module TranscoderExtensions - # Prevent the transcoder from modifying our meta hash - def initialize(file, options = {}, attachment = nil) - meta_value = attachment&.instance_read(:meta) - super - attachment&.instance_write(:meta, meta_value) - end - end -end - -Paperclip::Transcoder.prepend(Paperclip::TranscoderExtensions) diff --git a/lib/paperclip/video_transcoder.rb b/lib/paperclip/video_transcoder.rb deleted file mode 100644 index 4d9544231..000000000 --- a/lib/paperclip/video_transcoder.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Paperclip - # This transcoder is only to be used for the MediaAttachment model - # to check when uploaded videos are actually gifv's - class VideoTranscoder < Paperclip::Processor - def make - movie = FFMPEG::Movie.new(@file.path) - - attachment.instance.type = MediaAttachment.types[:gifv] unless movie.audio_codec - - Paperclip::Transcoder.make(file, actual_options(movie), attachment) - end - - private - - def actual_options(movie) - opts = options[:passthrough_options] - if opts && opts[:video_codecs].include?(movie.video_codec) && opts[:audio_codecs].include?(movie.audio_codec) && opts[:colorspaces].include?(movie.colorspace) - opts[:options] - else - options - end - end - end -end diff --git a/lib/terrapin/multi_pipe_extensions.rb b/lib/terrapin/multi_pipe_extensions.rb new file mode 100644 index 000000000..51d7de37c --- /dev/null +++ b/lib/terrapin/multi_pipe_extensions.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: false +# Fix adapted from https://github.com/thoughtbot/terrapin/pull/5 + +module Terrapin + module MultiPipeExtensions + def read + read_streams(@stdout_in, @stderr_in) + end + + def close_read + begin + @stdout_in.close + rescue IOError + # Do nothing + end + + begin + @stderr_in.close + rescue IOError + # Do nothing + end + end + + def read_streams(output, error) + @stdout_output = '' + @stderr_output = '' + + read_fds = [output, error] + + until read_fds.empty? + to_read, = IO.select(read_fds) + + if to_read.include?(output) + @stdout_output << read_stream(output) + read_fds.delete(output) if output.closed? + end + + if to_read.include?(error) + @stderr_output << read_stream(error) + read_fds.delete(error) if error.closed? + end + end + end + + def read_stream(io) + result = '' + + begin + while (partial_result = io.read_nonblock(8192)) + result << partial_result + end + rescue EOFError, Errno::EPIPE + io.close + rescue Errno::EINTR, Errno::EWOULDBLOCK, Errno::EAGAIN + # Do nothing + end + + result + end + end +end + +Terrapin::CommandLine::MultiPipe.prepend(Terrapin::MultiPipeExtensions) From 9aa7286c8fdbe3ad41123d313c9399976e3f3940 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 19 Mar 2021 11:07:56 +0100 Subject: [PATCH 16/70] Change deduplication order of tootctl maintenance fix-duplicates (#15923) Hopefully fixes #15922 Also update support up to latest database schema version --- lib/mastodon/maintenance_cli.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/mastodon/maintenance_cli.rb b/lib/mastodon/maintenance_cli.rb index 822051ceb..2c0f210c8 100644 --- a/lib/mastodon/maintenance_cli.rb +++ b/lib/mastodon/maintenance_cli.rb @@ -14,7 +14,7 @@ module Mastodon end MIN_SUPPORTED_VERSION = 2019_10_01_213028 - MAX_SUPPORTED_VERSION = 2020_12_18_054746 + MAX_SUPPORTED_VERSION = 2021_03_08_133107 # Stubs to enjoy ActiveRecord queries while not depending on a particular # version of the code/database @@ -142,7 +142,6 @@ module Mastodon @prompt.warn 'Please make sure to stop Mastodon and have a backup.' exit(1) unless @prompt.yes?('Continue?') - deduplicate_accounts! deduplicate_users! deduplicate_account_domain_blocks! deduplicate_account_identity_proofs! @@ -157,6 +156,7 @@ module Mastodon deduplicate_media_attachments! deduplicate_preview_cards! deduplicate_statuses! + deduplicate_accounts! deduplicate_tags! deduplicate_webauthn_credentials! From 2012c5ae5045be5ecfb888457c0857321b2e10c7 Mon Sep 17 00:00:00 2001 From: ThibG Date: Sat, 9 Jan 2021 03:51:32 +0100 Subject: [PATCH 17/70] Fix maintenance script not re-indexing some indexes on textual values (#15515) * Fix maintenance script not re-indexing some indexes on textual values Fixes #15475 * Refresh instance view at the end of the maintenance script run Co-authored-by: Claire --- lib/mastodon/maintenance_cli.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/mastodon/maintenance_cli.rb b/lib/mastodon/maintenance_cli.rb index 2c0f210c8..9f1eaf263 100644 --- a/lib/mastodon/maintenance_cli.rb +++ b/lib/mastodon/maintenance_cli.rb @@ -160,6 +160,7 @@ module Mastodon deduplicate_tags! deduplicate_webauthn_credentials! + Scenic.database.refresh_materialized_view('instances', concurrently: true, cascade: false) if ActiveRecord::Migrator.current_version >= 2020_12_06_004238 Rails.cache.clear @prompt.say 'Finished!' @@ -188,6 +189,11 @@ module Mastodon else ActiveRecord::Base.connection.add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true end + + @prompt.say 'Reindexing textual indexes on accounts…' + ActiveRecord::Base.connection.execute('REINDEX INDEX search_index;') + ActiveRecord::Base.connection.execute('REINDEX INDEX index_accounts_on_uri;') + ActiveRecord::Base.connection.execute('REINDEX INDEX index_accounts_on_url;') end def deduplicate_users! From 53814b2b3175291c35ad0fc98102752a7c8c773f Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 10 May 2021 17:31:55 +0200 Subject: [PATCH 18/70] Fix blocking someone not clearing up list feeds (#16205) --- app/lib/feed_manager.rb | 30 +++++++++++++++++++++ app/services/after_block_service.rb | 5 ++++ spec/services/after_block_service_spec.rb | 32 +++++++++++++++++++---- 3 files changed, 62 insertions(+), 5 deletions(-) diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index f0ad3e21f..ee735adfa 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -194,6 +194,36 @@ class FeedManager end end + # Clear all statuses from or mentioning target_account from a list feed + # @param [List] list + # @param [Account] target_account + # @return [void] + def clear_from_list(list, target_account) + timeline_key = key(:list, list.id) + timeline_status_ids = redis.zrange(timeline_key, 0, -1) + statuses = Status.where(id: timeline_status_ids).select(:id, :reblog_of_id, :account_id).to_a + reblogged_ids = Status.where(id: statuses.map(&:reblog_of_id).compact, account: target_account).pluck(:id) + with_mentions_ids = Mention.active.where(status_id: statuses.flat_map { |s| [s.id, s.reblog_of_id] }.compact, account: target_account).pluck(:status_id) + + target_statuses = statuses.select do |status| + status.account_id == target_account.id || reblogged_ids.include?(status.reblog_of_id) || with_mentions_ids.include?(status.id) || with_mentions_ids.include?(status.reblog_of_id) + end + + target_statuses.each do |status| + unpush_from_list(list, status) + end + end + + # Clear all statuses from or mentioning target_account from an account's lists + # @param [Account] account + # @param [Account] target_account + # @return [void] + def clear_from_lists(account, target_account) + List.where(account: account).each do |list| + clear_from_list(list, target_account) + end + end + # Populate home feed of account from scratch # @param [Account] account # @return [void] diff --git a/app/services/after_block_service.rb b/app/services/after_block_service.rb index 314919df8..899e84be4 100644 --- a/app/services/after_block_service.rb +++ b/app/services/after_block_service.rb @@ -6,6 +6,7 @@ class AfterBlockService < BaseService @target_account = target_account clear_home_feed! + clear_list_feeds! clear_notifications! clear_conversations! end @@ -16,6 +17,10 @@ class AfterBlockService < BaseService FeedManager.instance.clear_from_home(@account, @target_account) end + def clear_list_feeds! + FeedManager.instance.clear_from_lists(@account, @target_account) + end + def clear_conversations! AccountConversation.where(account: @account).where('? = ANY(participant_account_ids)', @target_account.id).in_batches.destroy_all end diff --git a/spec/services/after_block_service_spec.rb b/spec/services/after_block_service_spec.rb index f63b2045a..fe5b26b2b 100644 --- a/spec/services/after_block_service_spec.rb +++ b/spec/services/after_block_service_spec.rb @@ -5,12 +5,14 @@ RSpec.describe AfterBlockService, type: :service do -> { described_class.new.call(account, target_account) } end - let(:account) { Fabricate(:account) } - let(:target_account) { Fabricate(:account) } + let(:account) { Fabricate(:account) } + let(:target_account) { Fabricate(:account) } + let(:status) { Fabricate(:status, account: target_account) } + let(:other_status) { Fabricate(:status, account: target_account) } + let(:other_account_status) { Fabricate(:status) } + let(:other_account_reblog) { Fabricate(:status, reblog_of_id: other_status.id) } describe 'home timeline' do - let(:status) { Fabricate(:status, account: target_account) } - let(:other_account_status) { Fabricate(:status) } let(:home_timeline_key) { FeedManager.instance.key(:home, account.id) } before do @@ -20,10 +22,30 @@ RSpec.describe AfterBlockService, type: :service do it "clears account's statuses" do FeedManager.instance.push_to_home(account, status) FeedManager.instance.push_to_home(account, other_account_status) + FeedManager.instance.push_to_home(account, other_account_reblog) is_expected.to change { Redis.current.zrange(home_timeline_key, 0, -1) - }.from([status.id.to_s, other_account_status.id.to_s]).to([other_account_status.id.to_s]) + }.from([status.id.to_s, other_account_status.id.to_s, other_account_reblog.id.to_s]).to([other_account_status.id.to_s]) + end + end + + describe 'lists' do + let(:list) { Fabricate(:list, account: account) } + let(:list_timeline_key) { FeedManager.instance.key(:list, list.id) } + + before do + Redis.current.del(list_timeline_key) + end + + it "clears account's statuses" do + FeedManager.instance.push_to_list(list, status) + FeedManager.instance.push_to_list(list, other_account_status) + FeedManager.instance.push_to_list(list, other_account_reblog) + + is_expected.to change { + Redis.current.zrange(list_timeline_key, 0, -1) + }.from([status.id.to_s, other_account_status.id.to_s, other_account_reblog.id.to_s]).to([other_account_status.id.to_s]) end end end From 3799fd17ba5297ba0d88ee2c8c53d55d4665ff92 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 23 Apr 2021 22:51:21 +0200 Subject: [PATCH 19/70] Fix edge case where accepted follow cannot be processed because of follow limit (#16098) --- app/models/follow_request.rb | 2 +- spec/models/follow_request_spec.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/follow_request.rb b/app/models/follow_request.rb index 59fefcdf6..0b6f7629a 100644 --- a/app/models/follow_request.rb +++ b/app/models/follow_request.rb @@ -29,7 +29,7 @@ class FollowRequest < ApplicationRecord validates :account_id, uniqueness: { scope: :target_account_id } def authorize! - account.follow!(target_account, reblogs: show_reblogs, notify: notify, uri: uri) + account.follow!(target_account, reblogs: show_reblogs, notify: notify, uri: uri, bypass_limit: true) MergeWorker.perform_async(target_account.id, account.id) if account.local? destroy! end diff --git a/spec/models/follow_request_spec.rb b/spec/models/follow_request_spec.rb index cc484a5b9..36ce8ee60 100644 --- a/spec/models/follow_request_spec.rb +++ b/spec/models/follow_request_spec.rb @@ -7,7 +7,7 @@ RSpec.describe FollowRequest, type: :model do let(:target_account) { Fabricate(:account) } it 'calls Account#follow!, MergeWorker.perform_async, and #destroy!' do - expect(account).to receive(:follow!).with(target_account, reblogs: true, notify: false, uri: follow_request.uri) + expect(account).to receive(:follow!).with(target_account, reblogs: true, notify: false, uri: follow_request.uri, bypass_limit: true) expect(MergeWorker).to receive(:perform_async).with(target_account.id, account.id) expect(follow_request).to receive(:destroy!) follow_request.authorize! From 4bd8dc09d803d5b74e51030310bf7519e15783a8 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 16 Apr 2021 22:01:05 +0200 Subject: [PATCH 20/70] Fix reports of already suspended accounts being recorded (#16047) --- app/lib/activitypub/activity/flag.rb | 2 ++ app/services/report_service.rb | 2 ++ 2 files changed, 4 insertions(+) diff --git a/app/lib/activitypub/activity/flag.rb b/app/lib/activitypub/activity/flag.rb index 1659bc61f..7a5821064 100644 --- a/app/lib/activitypub/activity/flag.rb +++ b/app/lib/activitypub/activity/flag.rb @@ -10,6 +10,8 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity target_accounts.each do |target_account| target_statuses = target_statuses_by_account[target_account.id] + next if target_account.suspended? + ReportService.new.call( @account, target_account, diff --git a/app/services/report_service.rb b/app/services/report_service.rb index 9d9c7d6c9..bc0a8b464 100644 --- a/app/services/report_service.rb +++ b/app/services/report_service.rb @@ -10,6 +10,8 @@ class ReportService < BaseService @comment = options.delete(:comment) || '' @options = options + raise ActiveRecord::RecordNotFound if @target_account.suspended? + create_report! notify_staff! forward_to_origin! if !@target_account.local? && ActiveModel::Type::Boolean.new.cast(@options[:forward]) From 4b025cf7e6767665c2082184b47f2536aef70a91 Mon Sep 17 00:00:00 2001 From: ThibG Date: Thu, 21 Jan 2021 06:18:30 +0100 Subject: [PATCH 21/70] Fix sign-up restrictions based on IP addresses not being enforced (#15607) Fixes #15606 Co-authored-by: Claire --- app/models/user.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/user.rb b/app/models/user.rb index f8c8a6ab5..b4508c2eb 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -152,7 +152,7 @@ class User < ApplicationRecord def confirm new_user = !confirmed? - self.approved = true if open_registrations? + self.approved = true if open_registrations? && !sign_up_from_ip_requires_approval? super From 78d5bda97372b5cd22ff4a4233b6ba4a362c245d Mon Sep 17 00:00:00 2001 From: ThibG Date: Tue, 2 Feb 2021 14:49:57 +0100 Subject: [PATCH 22/70] Fix race conditions on account migration creation (#15597) * Atomically check for processing lock in Move handler * Prevent race condition when creating account migrations Fixes #15595 * Add tests Co-authored-by: Claire --- app/lib/activitypub/activity/move.rb | 11 +-- app/models/account_migration.rb | 14 ++- .../settings/migrations_controller_spec.rb | 37 ++++++- spec/lib/activitypub/activity/move_spec.rb | 99 ++++++++++++++----- 4 files changed, 127 insertions(+), 34 deletions(-) diff --git a/app/lib/activitypub/activity/move.rb b/app/lib/activitypub/activity/move.rb index 7e073f64d..8576ceccd 100644 --- a/app/lib/activitypub/activity/move.rb +++ b/app/lib/activitypub/activity/move.rb @@ -4,9 +4,8 @@ class ActivityPub::Activity::Move < ActivityPub::Activity PROCESSING_COOLDOWN = 7.days.seconds def perform - return if origin_account.uri != object_uri || processed? - - mark_as_processing! + return if origin_account.uri != object_uri + return unless mark_as_processing! target_account = ActivityPub::FetchRemoteAccountService.new.call(target_uri) @@ -35,12 +34,8 @@ class ActivityPub::Activity::Move < ActivityPub::Activity value_or_id(@json['target']) end - def processed? - redis.exists?("move_in_progress:#{@account.id}") - end - def mark_as_processing! - redis.setex("move_in_progress:#{@account.id}", PROCESSING_COOLDOWN, true) + redis.set("move_in_progress:#{@account.id}", true, nx: true, ex: PROCESSING_COOLDOWN) end def unmark_as_processing! diff --git a/app/models/account_migration.rb b/app/models/account_migration.rb index 4fae98ed7..ded32c9c6 100644 --- a/app/models/account_migration.rb +++ b/app/models/account_migration.rb @@ -14,6 +14,8 @@ # class AccountMigration < ApplicationRecord + include Redisable + COOLDOWN_PERIOD = 30.days.freeze belongs_to :account @@ -39,7 +41,13 @@ class AccountMigration < ApplicationRecord return false unless errors.empty? - save + RedisLock.acquire(lock_options) do |lock| + if lock.acquired? + save + else + raise Mastodon::RaceConditionError + end + end end def cooldown_at @@ -75,4 +83,8 @@ class AccountMigration < ApplicationRecord def validate_migration_cooldown errors.add(:base, I18n.t('migrations.errors.on_cooldown')) if account.migrations.within_cooldown.exists? end + + def lock_options + { redis: redis, key: "account_migration:#{account.id}" } + end end diff --git a/spec/controllers/settings/migrations_controller_spec.rb b/spec/controllers/settings/migrations_controller_spec.rb index 36e4ba86e..048d9de8d 100644 --- a/spec/controllers/settings/migrations_controller_spec.rb +++ b/spec/controllers/settings/migrations_controller_spec.rb @@ -51,7 +51,7 @@ describe Settings::MigrationsController do it_behaves_like 'authenticate user' end - context 'when user is sign in' do + context 'when user is signed in' do subject { post :create, params: { account_migration: { acct: acct, current_password: '12345678' } } } let(:user) { Fabricate(:user, password: '12345678') } @@ -67,12 +67,45 @@ describe Settings::MigrationsController do end end - context 'when acct is a current account' do + context 'when acct is the current account' do let(:acct) { user.account } it 'renders show' do is_expected.to render_template :show end + + it 'does not update the moved account' do + expect(user.account.reload.moved_to_account_id).to be_nil + end + end + + context 'when target account does not reference the account being moved from' do + let(:acct) { Fabricate(:account, also_known_as: []) } + + it 'renders show' do + is_expected.to render_template :show + end + + it 'does not update the moved account' do + expect(user.account.reload.moved_to_account_id).to be_nil + end + end + + context 'when a recent migration already exists ' do + let(:acct) { Fabricate(:account, also_known_as: [ActivityPub::TagManager.instance.uri_for(user.account)]) } + + before do + moved_to = Fabricate(:account, also_known_as: [ActivityPub::TagManager.instance.uri_for(user.account)]) + user.account.migrations.create!(acct: moved_to.acct) + end + + it 'renders show' do + is_expected.to render_template :show + end + + it 'does not update the moved account' do + expect(user.account.reload.moved_to_account_id).to be_nil + end end end end diff --git a/spec/lib/activitypub/activity/move_spec.rb b/spec/lib/activitypub/activity/move_spec.rb index 3574f273a..2d1d276c5 100644 --- a/spec/lib/activitypub/activity/move_spec.rb +++ b/spec/lib/activitypub/activity/move_spec.rb @@ -1,23 +1,11 @@ require 'rails_helper' RSpec.describe ActivityPub::Activity::Move do - let(:follower) { Fabricate(:account) } - let(:old_account) { Fabricate(:account) } - let(:new_account) { Fabricate(:account) } - - before do - follower.follow!(old_account) - - old_account.update!(uri: 'https://example.org/alice', domain: 'example.org', protocol: :activitypub, inbox_url: 'https://example.org/inbox') - new_account.update!(uri: 'https://example.com/alice', domain: 'example.com', protocol: :activitypub, inbox_url: 'https://example.com/inbox', also_known_as: [old_account.uri]) - - stub_request(:post, 'https://example.org/inbox').to_return(status: 200) - stub_request(:post, 'https://example.com/inbox').to_return(status: 200) - - service_stub = double - allow(ActivityPub::FetchRemoteAccountService).to receive(:new).and_return(service_stub) - allow(service_stub).to receive(:call).and_return(new_account) - end + let(:follower) { Fabricate(:account) } + let(:old_account) { Fabricate(:account, uri: 'https://example.org/alice', domain: 'example.org', protocol: :activitypub, inbox_url: 'https://example.org/inbox') } + let(:new_account) { Fabricate(:account, uri: 'https://example.com/alice', domain: 'example.com', protocol: :activitypub, inbox_url: 'https://example.com/inbox', also_known_as: also_known_as) } + let(:also_known_as) { [old_account.uri] } + let(:returned_account) { new_account } let(:json) do { @@ -30,6 +18,17 @@ RSpec.describe ActivityPub::Activity::Move do }.with_indifferent_access end + before do + follower.follow!(old_account) + + stub_request(:post, old_account.inbox_url).to_return(status: 200) + stub_request(:post, new_account.inbox_url).to_return(status: 200) + + service_stub = double + allow(ActivityPub::FetchRemoteAccountService).to receive(:new).and_return(service_stub) + allow(service_stub).to receive(:call).and_return(returned_account) + end + describe '#perform' do subject { described_class.new(json, old_account) } @@ -37,16 +36,70 @@ RSpec.describe ActivityPub::Activity::Move do subject.perform end - it 'sets moved account on old account' do - expect(old_account.reload.moved_to_account_id).to eq new_account.id + context 'when all conditions are met' do + it 'sets moved account on old account' do + expect(old_account.reload.moved_to_account_id).to eq new_account.id + end + + it 'makes followers unfollow old account' do + expect(follower.following?(old_account)).to be false + end + + it 'makes followers follow-request the new account' do + expect(follower.requested?(new_account)).to be true + end end - it 'makes followers unfollow old account' do - expect(follower.following?(old_account)).to be false + context "when the new account can't be resolved" do + let(:returned_account) { nil } + + it 'does not set moved account on old account' do + expect(old_account.reload.moved_to_account_id).to be_nil + end + + it 'does not make followers unfollow old account' do + expect(follower.following?(old_account)).to be true + end + + it 'does not make followers follow-request the new account' do + expect(follower.requested?(new_account)).to be false + end end - it 'makes followers follow-request the new account' do - expect(follower.requested?(new_account)).to be true + context 'when the new account does not references the old account' do + let(:also_known_as) { [] } + + it 'does not set moved account on old account' do + expect(old_account.reload.moved_to_account_id).to be_nil + end + + it 'does not make followers unfollow old account' do + expect(follower.following?(old_account)).to be true + end + + it 'does not make followers follow-request the new account' do + expect(follower.requested?(new_account)).to be false + end + end + + context 'when a Move has been recently processed' do + around do |example| + Redis.current.set("move_in_progress:#{old_account.id}", true, nx: true, ex: 7.days.seconds) + example.run + Redis.current.del("move_in_progress:#{old_account.id}") + end + + it 'does not set moved account on old account' do + expect(old_account.reload.moved_to_account_id).to be_nil + end + + it 'does not make followers unfollow old account' do + expect(follower.following?(old_account)).to be true + end + + it 'does not make followers follow-request the new account' do + expect(follower.requested?(new_account)).to be false + end end end end From 3a4d9f1f2df95d351b8692a9895c78c637440cd5 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 12 Mar 2021 05:25:50 +0100 Subject: [PATCH 23/70] =?UTF-8?q?Fix=20not=20being=20able=20to=20change=20?= =?UTF-8?q?world=20filter=20expiration=20back=20to=20=E2=80=9CNever?= =?UTF-8?q?=E2=80=9D=20(#15858)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #15849 --- app/models/concerns/expireable.rb | 2 +- app/views/filters/_fields.html.haml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/concerns/expireable.rb b/app/models/concerns/expireable.rb index a66a4661b..4d902abcb 100644 --- a/app/models/concerns/expireable.rb +++ b/app/models/concerns/expireable.rb @@ -17,7 +17,7 @@ module Expireable end def expires_in=(interval) - self.expires_at = interval.to_i.seconds.from_now if interval.present? + self.expires_at = interval.present? ? interval.to_i.seconds.from_now : nil @expires_in = interval end diff --git a/app/views/filters/_fields.html.haml b/app/views/filters/_fields.html.haml index fb94a07fc..84dcdcca5 100644 --- a/app/views/filters/_fields.html.haml +++ b/app/views/filters/_fields.html.haml @@ -2,7 +2,7 @@ .fields-row__column.fields-row__column-6.fields-group = f.input :phrase, as: :string, wrapper: :with_label, hint: false .fields-row__column.fields-row__column-6.fields-group - = f.input :expires_in, wrapper: :with_label, collection: [30.minutes, 1.hour, 6.hours, 12.hours, 1.day, 1.week].map(&:to_i), label_method: lambda { |i| I18n.t("invites.expires_in.#{i}") }, prompt: I18n.t('invites.expires_in_prompt') + = f.input :expires_in, wrapper: :with_label, collection: [30.minutes, 1.hour, 6.hours, 12.hours, 1.day, 1.week].map(&:to_i), label_method: lambda { |i| I18n.t("invites.expires_in.#{i}") }, include_blank: I18n.t('invites.expires_in_prompt') .fields-group = f.input :context, wrapper: :with_block_label, collection: CustomFilter::VALID_CONTEXTS, as: :check_boxes, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li', label_method: lambda { |context| I18n.t("filters.contexts.#{context}") }, include_blank: false From 0fe57a91407ce325e02a4778b967f13dbc33ed43 Mon Sep 17 00:00:00 2001 From: Levi Bard Date: Sun, 10 Jan 2021 12:47:21 +0100 Subject: [PATCH 24/70] Fix muting users with duration via the REST api (#15516) --- app/controllers/api/v1/accounts_controller.rb | 2 +- .../api/v1/accounts_controller_spec.rb | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/v1/accounts_controller.rb b/app/controllers/api/v1/accounts_controller.rb index 3e66ff212..953874e1a 100644 --- a/app/controllers/api/v1/accounts_controller.rb +++ b/app/controllers/api/v1/accounts_controller.rb @@ -42,7 +42,7 @@ class Api::V1::AccountsController < Api::BaseController end def mute - MuteService.new.call(current_user.account, @account, notifications: truthy_param?(:notifications), duration: (params[:duration] || 0)) + MuteService.new.call(current_user.account, @account, notifications: truthy_param?(:notifications), duration: (params[:duration]&.to_i || 0)) render json: @account, serializer: REST::RelationshipSerializer, relationships: relationships end diff --git a/spec/controllers/api/v1/accounts_controller_spec.rb b/spec/controllers/api/v1/accounts_controller_spec.rb index 1e656503f..d9ee37ffa 100644 --- a/spec/controllers/api/v1/accounts_controller_spec.rb +++ b/spec/controllers/api/v1/accounts_controller_spec.rb @@ -268,6 +268,34 @@ RSpec.describe Api::V1::AccountsController, type: :controller do it_behaves_like 'forbidden for wrong scope', 'read:accounts' end + describe 'POST #mute with nonzero duration set' do + let(:scopes) { 'write:mutes' } + let(:other_account) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob')).account } + + before do + user.account.follow!(other_account) + post :mute, params: { id: other_account.id, duration: 300 } + end + + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'does not remove the following relation between user and target user' do + expect(user.account.following?(other_account)).to be true + end + + it 'creates a muting relation' do + expect(user.account.muting?(other_account)).to be true + end + + it 'mutes notifications' do + expect(user.account.muting_notifications?(other_account)).to be true + end + + it_behaves_like 'forbidden for wrong scope', 'read:accounts' + end + describe 'POST #unmute' do let(:scopes) { 'write:mutes' } let(:other_account) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob')).account } From 9437e9f0b956ac92b1403faa4009b7e5ce121481 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 19 Mar 2021 20:23:08 +0100 Subject: [PATCH 25/70] Fix custom CSS when CDN_HOST is set (#15927) --- app/views/layouts/application.html.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/layouts/application.html.haml b/app/views/layouts/application.html.haml index 9501207e0..436024ee3 100755 --- a/app/views/layouts/application.html.haml +++ b/app/views/layouts/application.html.haml @@ -31,7 +31,7 @@ = stylesheet_link_tag '/inert.css', skip_pipeline: true, media: 'all', id: 'inert-style' - if Setting.custom_css.present? - = stylesheet_link_tag custom_css_path, media: 'all' + = stylesheet_link_tag custom_css_path, host: request.host, media: 'all' = yield :header_tags From add7b9f82e124f88eef4e48fa897547d1a0d8a09 Mon Sep 17 00:00:00 2001 From: ThibG Date: Fri, 29 Jan 2021 18:38:56 +0100 Subject: [PATCH 26/70] =?UTF-8?q?Fix=20=E2=80=9Ctootctl=20accounts=20unfol?= =?UTF-8?q?low=E2=80=9D=20(#15639)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #15635 Co-authored-by: Claire --- lib/mastodon/accounts_cli.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb index 653bfca30..74162256f 100644 --- a/lib/mastodon/accounts_cli.rb +++ b/lib/mastodon/accounts_cli.rb @@ -402,7 +402,7 @@ module Mastodon exit(1) end - parallelize_with_progress(target_account.followers.local) do |account| + processed, = parallelize_with_progress(target_account.followers.local) do |account| UnfollowService.new.call(account, target_account) end From fc4b9856f8cfcb1471e69a261c930b2900a0646e Mon Sep 17 00:00:00 2001 From: Sophie Parker Date: Thu, 31 Dec 2020 09:19:55 +1100 Subject: [PATCH 27/70] Improve Emoji import (fix #15429) (#15430) * Improve Emoji import Skip macOS '._' shadow files in tar archive to speed up import * Fix codeclimate format issue with whitespace * Update lib/mastodon/emoji_cli.rb suggestions from Gargron to improve comment Co-authored-by: Eugen Rochko * Update emoji_cli.rb Remove extraneous comment (macOS-specific comment now with correct line) Co-authored-by: Eugen Rochko --- lib/mastodon/emoji_cli.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/mastodon/emoji_cli.rb b/lib/mastodon/emoji_cli.rb index da8fd6a0d..0a1f538e6 100644 --- a/lib/mastodon/emoji_cli.rb +++ b/lib/mastodon/emoji_cli.rb @@ -43,7 +43,12 @@ module Mastodon tar.each do |entry| next unless entry.file? && entry.full_name.end_with?('.png') - shortcode = [options[:prefix], File.basename(entry.full_name, '.*'), options[:suffix]].compact.join + filename = File.basename(entry.full_name, '.*') + + # Skip macOS shadow files + next if filename.start_with?('._') + + shortcode = [options[:prefix], filename, options[:suffix]].compact.join custom_emoji = CustomEmoji.local.find_by(shortcode: shortcode) if custom_emoji && !options[:overwrite] From 0951c691ff7b1e4de515613b8fc7dc67894c0f30 Mon Sep 17 00:00:00 2001 From: Stanislas Date: Wed, 24 Mar 2021 10:55:16 +0100 Subject: [PATCH 28/70] tootctl emoji import: case insensitive duplicate check (#15738) --- lib/mastodon/emoji_cli.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mastodon/emoji_cli.rb b/lib/mastodon/emoji_cli.rb index 0a1f538e6..5bee70ea5 100644 --- a/lib/mastodon/emoji_cli.rb +++ b/lib/mastodon/emoji_cli.rb @@ -49,7 +49,7 @@ module Mastodon next if filename.start_with?('._') shortcode = [options[:prefix], filename, options[:suffix]].compact.join - custom_emoji = CustomEmoji.local.find_by(shortcode: shortcode) + custom_emoji = CustomEmoji.local.find_by("LOWER(shortcode) = ?", shortcode.downcase) if custom_emoji && !options[:overwrite] skipped += 1 From 4978d387ee33062768910bfea70902b2778fa5fa Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 19 Mar 2021 17:15:36 +0100 Subject: [PATCH 29/70] WIP (#15222) --- app/models/concerns/omniauthable.rb | 1 - app/models/user.rb | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/models/concerns/omniauthable.rb b/app/models/concerns/omniauthable.rb index 4ea219537..d6b414148 100644 --- a/app/models/concerns/omniauthable.rb +++ b/app/models/concerns/omniauthable.rb @@ -68,7 +68,6 @@ module Omniauthable def user_params_from_auth(email, auth) { email: email || "#{TEMP_EMAIL_PREFIX}-#{auth.uid}-#{auth.provider}.com", - password: Devise.friendly_token[0, 20], agreement: true, external: true, account_attributes: { diff --git a/app/models/user.rb b/app/models/user.rb index b4508c2eb..5a149f573 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -468,7 +468,7 @@ class User < ApplicationRecord end def validate_email_dns? - email_changed? && !(Rails.env.test? || Rails.env.development?) + email_changed? && !external? && !(Rails.env.test? || Rails.env.development?) end def invite_text_required? From 07042a0913bdb28da5d5ead3c7ecf310bb9c6acd Mon Sep 17 00:00:00 2001 From: kaiyou Date: Fri, 8 Jan 2021 07:07:08 +0100 Subject: [PATCH 30/70] Support clock drift in Omniauth SAML provider (#15511) The setting is not well documented by the provider, but allows for clock skew between SP and IDP, see: https://github.com/omniauth/omniauth-saml/blob/master/spec/omniauth/strategies/saml_spec.rb Co-authored-by: kaiyou --- config/initializers/omniauth.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/config/initializers/omniauth.rb b/config/initializers/omniauth.rb index e8d7697a1..9e037f421 100644 --- a/config/initializers/omniauth.rb +++ b/config/initializers/omniauth.rb @@ -60,6 +60,7 @@ Devise.setup do |config| saml_options[:attribute_statements][:verified] = [ENV['SAML_ATTRIBUTES_STATEMENTS_VERIFIED']] if ENV['SAML_ATTRIBUTES_STATEMENTS_VERIFIED'] saml_options[:attribute_statements][:verified_email] = [ENV['SAML_ATTRIBUTES_STATEMENTS_VERIFIED_EMAIL']] if ENV['SAML_ATTRIBUTES_STATEMENTS_VERIFIED_EMAIL'] saml_options[:uid_attribute] = ENV['SAML_UID_ATTRIBUTE'] if ENV['SAML_UID_ATTRIBUTE'] + saml_options[:allowed_clock_drift] = ENV['SAML_ALLOWED_CLOCK_DRIFT'] if ENV['SAML_ALLOWED_CLOCK_DRIFT'] config.omniauth :saml, saml_options end end From 24dee67d32f51aeb6db50371df682afa8a2b108a Mon Sep 17 00:00:00 2001 From: Claire Date: Tue, 9 Feb 2021 18:12:54 +0100 Subject: [PATCH 31/70] Create instance actor if it hasn't been properly seeded (#15693) An uncommon but somewhat difficult to digagnose issue is dealing with improperly-seeded databases. In such cases, instance-signed fetches will fail with a ActiveRecord::RecordNotFound error, usually caught and handled as generic 404, leading people to think the remote resource itself has not been found, while it's the local instance actor that does not exist. This commit changes the code so that failure to find the instance actor automatically creates a new one, so that improperly-seeded databases do not cause any issue. --- app/controllers/instance_actors_controller.rb | 2 +- app/models/concerns/account_finder_concern.rb | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/controllers/instance_actors_controller.rb b/app/controllers/instance_actors_controller.rb index 4b074ca19..b3b5476e2 100644 --- a/app/controllers/instance_actors_controller.rb +++ b/app/controllers/instance_actors_controller.rb @@ -13,7 +13,7 @@ class InstanceActorsController < ApplicationController private def set_account - @account = Account.find(-99) + @account = Account.representative end def restrict_fields_to diff --git a/app/models/concerns/account_finder_concern.rb b/app/models/concerns/account_finder_concern.rb index 04b2c981b..0dadddad1 100644 --- a/app/models/concerns/account_finder_concern.rb +++ b/app/models/concerns/account_finder_concern.rb @@ -14,6 +14,8 @@ module AccountFinderConcern def representative Account.find(-99) + rescue ActiveRecord::RecordNotFound + Account.create!(id: -99, actor_type: 'Application', locked: true, username: Rails.configuration.x.local_domain) end def find_local(username) From a319fd3cc4ae352fdd6363cf01a2111c1240d684 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 15 Apr 2021 16:28:43 +0200 Subject: [PATCH 32/70] Fix app name, website and redirect URIs not having a maximum length (#16042) Fix app scopes not being validated --- app/lib/application_extension.rb | 4 +- config/initializers/doorkeeper.rb | 5 ++ .../api/v1/apps_controller_spec.rb | 78 ++++++++++++++++--- 3 files changed, 77 insertions(+), 10 deletions(-) diff --git a/app/lib/application_extension.rb b/app/lib/application_extension.rb index 1d80b8c6d..e61cd0721 100644 --- a/app/lib/application_extension.rb +++ b/app/lib/application_extension.rb @@ -4,6 +4,8 @@ module ApplicationExtension extend ActiveSupport::Concern included do - validates :website, url: true, if: :website? + validates :name, length: { maximum: 60 } + validates :website, url: true, length: { maximum: 2_000 }, if: :website? + validates :redirect_uri, length: { maximum: 2_000 } end end diff --git a/config/initializers/doorkeeper.rb b/config/initializers/doorkeeper.rb index 63cff7c59..f78db8653 100644 --- a/config/initializers/doorkeeper.rb +++ b/config/initializers/doorkeeper.rb @@ -52,6 +52,11 @@ Doorkeeper.configure do # Issue access tokens with refresh token (disabled by default) # use_refresh_token + # Forbids creating/updating applications with arbitrary scopes that are + # not in configuration, i.e. `default_scopes` or `optional_scopes`. + # (Disabled by default) + enforce_configured_scopes + # Provide support for an owner to be assigned to each registered application (disabled by default) # Optional parameter :confirmation => true (default false) if you want to enforce ownership of # a registered application diff --git a/spec/controllers/api/v1/apps_controller_spec.rb b/spec/controllers/api/v1/apps_controller_spec.rb index 60a4c3b41..70cd62d48 100644 --- a/spec/controllers/api/v1/apps_controller_spec.rb +++ b/spec/controllers/api/v1/apps_controller_spec.rb @@ -4,23 +4,83 @@ RSpec.describe Api::V1::AppsController, type: :controller do render_views describe 'POST #create' do + let(:client_name) { 'Test app' } + let(:scopes) { nil } + let(:redirect_uris) { 'urn:ietf:wg:oauth:2.0:oob' } + let(:website) { nil } + + let(:app_params) do + { + client_name: client_name, + redirect_uris: redirect_uris, + scopes: scopes, + website: website, + } + end + before do - post :create, params: { client_name: 'Test app', redirect_uris: 'urn:ietf:wg:oauth:2.0:oob' } + post :create, params: app_params end - it 'returns http success' do - expect(response).to have_http_status(200) + context 'with valid params' do + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'creates an OAuth app' do + expect(Doorkeeper::Application.find_by(name: client_name)).to_not be nil + end + + it 'returns client ID and client secret' do + json = body_as_json + + expect(json[:client_id]).to_not be_blank + expect(json[:client_secret]).to_not be_blank + end end - it 'creates an OAuth app' do - expect(Doorkeeper::Application.find_by(name: 'Test app')).to_not be nil + context 'with an unsupported scope' do + let(:scopes) { 'hoge' } + + it 'returns http unprocessable entity' do + expect(response).to have_http_status(422) + end end - it 'returns client ID and client secret' do - json = body_as_json + context 'with many duplicate scopes' do + let(:scopes) { (%w(read) * 40).join(' ') } - expect(json[:client_id]).to_not be_blank - expect(json[:client_secret]).to_not be_blank + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'only saves the scope once' do + expect(Doorkeeper::Application.find_by(name: client_name).scopes.to_s).to eq 'read' + end + end + + context 'with a too-long name' do + let(:client_name) { 'hoge' * 20 } + + it 'returns http unprocessable entity' do + expect(response).to have_http_status(422) + end + end + + context 'with a too-long website' do + let(:website) { 'https://foo.bar/' + ('hoge' * 2_000) } + + it 'returns http unprocessable entity' do + expect(response).to have_http_status(422) + end + end + + context 'with a too-long redirect_uris' do + let(:redirect_uris) { 'https://foo.bar/' + ('hoge' * 2_000) } + + it 'returns http unprocessable entity' do + expect(response).to have_http_status(422) + end end end end From c89809afc5847e9208dfdebe4bbe5c10b5b8d5d2 Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 31 May 2021 22:59:30 +0200 Subject: [PATCH 33/70] Fix some IDs in instance actor outbox (#16343) --- app/controllers/activitypub/outboxes_controller.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/activitypub/outboxes_controller.rb b/app/controllers/activitypub/outboxes_controller.rb index 5fd735ad6..338b703a6 100644 --- a/app/controllers/activitypub/outboxes_controller.rb +++ b/app/controllers/activitypub/outboxes_controller.rb @@ -29,7 +29,7 @@ class ActivityPub::OutboxesController < ActivityPub::BaseController ) else ActivityPub::CollectionPresenter.new( - id: account_outbox_url(@account), + id: outbox_url, type: :ordered, size: @account.statuses_count, first: outbox_url(page: true), @@ -47,11 +47,11 @@ class ActivityPub::OutboxesController < ActivityPub::BaseController end def next_page - account_outbox_url(@account, page: true, max_id: @statuses.last.id) if @statuses.size == LIMIT + outbox_url(page: true, max_id: @statuses.last.id) if @statuses.size == LIMIT end def prev_page - account_outbox_url(@account, page: true, min_id: @statuses.first.id) unless @statuses.empty? + outbox_url(page: true, min_id: @statuses.first.id) unless @statuses.empty? end def set_statuses From 678e0ad419bf6e4c5d47c568e0b08189b4925226 Mon Sep 17 00:00:00 2001 From: Jeong Arm Date: Mon, 31 May 2021 00:57:47 +0900 Subject: [PATCH 34/70] Remove set-cookie header on custom.css (#16314) * Remove set-cookie header on custom.css * Additional fix for set-cookie --- app/controllers/custom_css_controller.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/controllers/custom_css_controller.rb b/app/controllers/custom_css_controller.rb index 0a667a6a6..e1dc5eaf6 100644 --- a/app/controllers/custom_css_controller.rb +++ b/app/controllers/custom_css_controller.rb @@ -3,11 +3,16 @@ class CustomCssController < ApplicationController skip_before_action :store_current_location skip_before_action :require_functional! + skip_before_action :update_user_sign_in + skip_before_action :set_session_activity + + skip_around_action :set_locale before_action :set_cache_headers def show expires_in 3.minutes, public: true + request.session_options[:skip] = true render plain: Setting.custom_css || '', content_type: 'text/css' end end From 4d41c91335f74d3b2e4535526b245e933c8fffd3 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 19 May 2021 23:52:08 +0200 Subject: [PATCH 35/70] Fix some RedisLocks auto-releasing too fast (#16276) * Fix Delete and Create-related locks expiring too fast Fixes #16238 By default, RedisLock expires after 10 seconds, which may not be enough to process statuses, especially when those have attached media files. This commit extends those 10 seconds to 15 minutes, which should be plenty enough to handle any status, while being short enough to not waste many sidekiq job retries in the exceedingly rare case in which a sidekiq process would crash when processing a `Create` or `Delete`. * Fix other RedisLock autorelease durations Fixes #15645 - things that only perform a few simple database queries (e.g. finding and saving a record) have been left unchanged, so they'll still use the default 10s duration - things that perform significantly more complex database queries have been changed to a 5 minutes timeout - things that perform multiple HTTP queries have been changed to a 15 minutes timeout --- app/lib/activitypub/activity.rb | 4 ++-- app/services/activitypub/process_account_service.rb | 2 +- app/services/fetch_link_card_service.rb | 2 +- app/services/remove_status_service.rb | 2 +- app/services/resolve_account_service.rb | 2 +- app/workers/distribution_worker.rb | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index 3baee4ca4..d2ec122a4 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -216,8 +216,8 @@ class ActivityPub::Activity redis.del(key) end - def lock_or_fail(key) - RedisLock.acquire({ redis: Redis.current, key: key }) do |lock| + def lock_or_fail(key, expire_after = 15.minutes.seconds) + RedisLock.acquire({ redis: Redis.current, key: key, autorelease: expire_after }) do |lock| if lock.acquired? yield else diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index 6afeb92d6..0d52a3b84 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -281,7 +281,7 @@ class ActivityPub::ProcessAccountService < BaseService end def lock_options - { redis: Redis.current, key: "process_account:#{@uri}" } + { redis: Redis.current, key: "process_account:#{@uri}", autorelease: 15.minutes.seconds } end def process_tags diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index 7efa31054..661641070 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -174,6 +174,6 @@ class FetchLinkCardService < BaseService end def lock_options - { redis: Redis.current, key: "fetch:#{@url}" } + { redis: Redis.current, key: "fetch:#{@url}", autorelease: 15.minutes.seconds } end end diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index 52d3f108c..e17b77377 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -170,6 +170,6 @@ class RemoveStatusService < BaseService end def lock_options - { redis: Redis.current, key: "distribute:#{@status.id}" } + { redis: Redis.current, key: "distribute:#{@status.id}", autorelease: 5.minutes.seconds } end end diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb index 3301aaf51..c78acaea8 100644 --- a/app/services/resolve_account_service.rb +++ b/app/services/resolve_account_service.rb @@ -145,6 +145,6 @@ class ResolveAccountService < BaseService end def lock_options - { redis: Redis.current, key: "resolve:#{@username}@#{@domain}" } + { redis: Redis.current, key: "resolve:#{@username}@#{@domain}", autorelease: 15.minutes.seconds } end end diff --git a/app/workers/distribution_worker.rb b/app/workers/distribution_worker.rb index 4e20ef31b..e85cd7e95 100644 --- a/app/workers/distribution_worker.rb +++ b/app/workers/distribution_worker.rb @@ -4,7 +4,7 @@ class DistributionWorker include Sidekiq::Worker def perform(status_id) - RedisLock.acquire(redis: Redis.current, key: "distribute:#{status_id}") do |lock| + RedisLock.acquire(redis: Redis.current, key: "distribute:#{status_id}", autorelease: 5.minutes.seconds) do |lock| if lock.acquired? FanOutOnWriteService.new.call(Status.find(status_id)) else From 31d9aa8ed02298aadee86e6f61af6214249c06ac Mon Sep 17 00:00:00 2001 From: Claire Date: Sat, 22 May 2021 15:00:33 +0200 Subject: [PATCH 36/70] Fix media proxy RedisLocks auto-releasing too fast (#16291) Follow-up to #16276 --- app/controllers/media_proxy_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/media_proxy_controller.rb b/app/controllers/media_proxy_controller.rb index 0b1d09de9..5807cc6fc 100644 --- a/app/controllers/media_proxy_controller.rb +++ b/app/controllers/media_proxy_controller.rb @@ -45,7 +45,7 @@ class MediaProxyController < ApplicationController end def lock_options - { redis: Redis.current, key: "media_download:#{params[:id]}" } + { redis: Redis.current, key: "media_download:#{params[:id]}", autorelease: 15.minutes.seconds } end def reject_media? From 1b32c001bc7c1ba4560801e5b8e7c09f33ede226 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 2 Jun 2021 19:15:17 +0200 Subject: [PATCH 37/70] Fix migration script not being able to run if it fails midway (#16312) * Fix migration script not being able to run if it fails midway * Fix old migration script * Fix old migration script * Refactor CorruptionError --- ...175042_add_case_insensitive_index_to_tags.rb | 8 +++++++- ...023_add_fixed_lowercase_index_to_accounts.rb | 17 ++++++----------- lib/mastodon/migration_helpers.rb | 14 ++++++++++++++ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb index 057fc86ba..ceb8e563b 100644 --- a/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb +++ b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb @@ -15,7 +15,13 @@ class AddCaseInsensitiveIndexToTags < ActiveRecord::Migration[5.2] Tag.where(id: redundant_tag_ids).in_batches.delete_all end - safety_assured { execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower ON tags (lower(name))' } + begin + safety_assured { execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower ON tags (lower(name))' } + rescue ActiveRecord::StatementInvalid + remove_index :tags, name: 'index_tags_on_name_lower' + raise + end + remove_index :tags, name: 'index_tags_on_name' remove_index :tags, name: 'hashtag_search_index' end diff --git a/db/migrate/20200620164023_add_fixed_lowercase_index_to_accounts.rb b/db/migrate/20200620164023_add_fixed_lowercase_index_to_accounts.rb index c3aa8e33c..366bf9aa7 100644 --- a/db/migrate/20200620164023_add_fixed_lowercase_index_to_accounts.rb +++ b/db/migrate/20200620164023_add_fixed_lowercase_index_to_accounts.rb @@ -1,16 +1,10 @@ +require Rails.root.join('lib', 'mastodon', 'migration_helpers') + class AddFixedLowercaseIndexToAccounts < ActiveRecord::Migration[5.2] + include Mastodon::MigrationHelpers + disable_ddl_transaction! - class CorruptionError < StandardError - def cause - nil - end - - def backtrace - [] - end - end - def up if index_name_exists?(:accounts, 'old_index_accounts_on_username_and_domain_lower') && index_name_exists?(:accounts, 'index_accounts_on_username_and_domain_lower') remove_index :accounts, name: 'index_accounts_on_username_and_domain_lower' @@ -21,7 +15,8 @@ class AddFixedLowercaseIndexToAccounts < ActiveRecord::Migration[5.2] begin add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true, algorithm: :concurrently rescue ActiveRecord::RecordNotUnique - raise CorruptionError, 'Migration failed because of index corruption, see https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/#fixing' + remove_index :accounts, name: 'index_accounts_on_username_and_domain_lower' + raise CorruptionError end remove_index :accounts, name: 'old_index_accounts_on_username_and_domain_lower' if index_name_exists?(:accounts, 'old_index_accounts_on_username_and_domain_lower') diff --git a/lib/mastodon/migration_helpers.rb b/lib/mastodon/migration_helpers.rb index bf2314ecb..30645b923 100644 --- a/lib/mastodon/migration_helpers.rb +++ b/lib/mastodon/migration_helpers.rb @@ -41,6 +41,20 @@ module Mastodon module MigrationHelpers + class CorruptionError < StandardError + def initialize(message = nil) + super(message.presence || 'Migration failed because of index corruption, see https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/#fixing') + end + + def cause + nil + end + + def backtrace + [] + end + end + # Stub for Database.postgresql? from GitLab def self.postgresql? ActiveRecord::Base.configurations[Rails.env]['adapter'].casecmp('postgresql').zero? From 1cfa2bdb03426a39c42e910eec0820064e2e998c Mon Sep 17 00:00:00 2001 From: Holger Date: Thu, 9 Sep 2021 05:33:36 +0800 Subject: [PATCH 38/70] use relative path for `scope` (#16714) Use relative path for `scope` in web manifest to allow users use PWA correctly via alternate domains. --- app/serializers/manifest_serializer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/serializers/manifest_serializer.rb b/app/serializers/manifest_serializer.rb index dafe8f55b..4786aa760 100644 --- a/app/serializers/manifest_serializer.rb +++ b/app/serializers/manifest_serializer.rb @@ -48,7 +48,7 @@ class ManifestSerializer < ActiveModel::Serializer end def scope - root_url + '/' end def share_target From 3f882c2c170d76c0f777f0e45373e80220c00c09 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 14 Oct 2021 19:59:21 +0200 Subject: [PATCH 39/70] Fix scheduled statuses decreasing statuses counts (#16791) * Add tests * Fix scheduled statuses decreasing statuses counts Fixes #16774 --- app/models/status.rb | 2 +- app/services/post_status_service.rb | 3 ++ spec/services/post_status_service_spec.rb | 42 +++++++++++++---------- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/app/models/status.rb b/app/models/status.rb index 4353d2710..b713626c2 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -422,7 +422,7 @@ class Status < ApplicationRecord end def decrement_counter_caches - return if direct_visibility? + return if direct_visibility? || new_record? account&.decrement_count!(:statuses_count) reblog&.decrement_count!(:reblogs_count) if reblog? diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb index 0a383d6a3..85aaec4d6 100644 --- a/app/services/post_status_service.rb +++ b/app/services/post_status_service.rb @@ -74,6 +74,9 @@ class PostStatusService < BaseService status_for_validation = @account.statuses.build(status_attributes) if status_for_validation.valid? + # Marking the status as destroyed is necessary to prevent the status from being + # persisted when the associated media attachments get updated when creating the + # scheduled status. status_for_validation.destroy # The following transaction block is needed to wrap the UPDATEs to diff --git a/spec/services/post_status_service_spec.rb b/spec/services/post_status_service_spec.rb index 147a59fc3..d21270c79 100644 --- a/spec/services/post_status_service_spec.rb +++ b/spec/services/post_status_service_spec.rb @@ -25,29 +25,33 @@ RSpec.describe PostStatusService, type: :service do expect(status.thread).to eq in_reply_to_status end - it 'schedules a status' do - account = Fabricate(:account) - future = Time.now.utc + 2.hours + context 'when scheduling a status' do + let!(:account) { Fabricate(:account) } + let!(:future) { Time.now.utc + 2.hours } + let!(:previous_status) { Fabricate(:status, account: account) } - status = subject.call(account, text: 'Hi future!', scheduled_at: future) + it 'schedules a status' do + status = subject.call(account, text: 'Hi future!', scheduled_at: future) + expect(status).to be_a ScheduledStatus + expect(status.scheduled_at).to eq future + expect(status.params['text']).to eq 'Hi future!' + end - expect(status).to be_a ScheduledStatus - expect(status.scheduled_at).to eq future - expect(status.params['text']).to eq 'Hi future!' - end + it 'does not immediately create a status' do + media = Fabricate(:media_attachment, account: account) + status = subject.call(account, text: 'Hi future!', media_ids: [media.id], scheduled_at: future) - it 'does not immediately create a status when scheduling a status' do - account = Fabricate(:account) - media = Fabricate(:media_attachment) - future = Time.now.utc + 2.hours + expect(status).to be_a ScheduledStatus + expect(status.scheduled_at).to eq future + expect(status.params['text']).to eq 'Hi future!' + expect(status.params['media_ids']).to eq [media.id] + expect(media.reload.status).to be_nil + expect(Status.where(text: 'Hi future!').exists?).to be_falsey + end - status = subject.call(account, text: 'Hi future!', media_ids: [media.id], scheduled_at: future) - - expect(status).to be_a ScheduledStatus - expect(status.scheduled_at).to eq future - expect(status.params['text']).to eq 'Hi future!' - expect(media.reload.status).to be_nil - expect(Status.where(text: 'Hi future!').exists?).to be_falsey + it 'does not change statuses count' do + expect { subject.call(account, text: 'Hi future!', scheduled_at: future, thread: previous_status) }.not_to change { [account.statuses_count, previous_status.replies_count] } + end end it 'creates response to the original status of boost' do From 53e4efd07d14e576b33aaddf6b6b4acc85dcbbc1 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 11 Aug 2021 17:48:42 +0200 Subject: [PATCH 40/70] Fix followers synchronization mechanism not working when URI has empty path (#16510) * Fix followers synchronization mechanism not working when URI has empty path To my knowledge, there is no current implementation on the fediverse that can use bare domains (e.g., actor is at https://example.org instead of something like https://example.org/actor) that also plans to support the followers synchronization mechanism. However, Mastodon's current implementation would exclude such accounts from followers list. Also adds tests and rename them to reflect the proper method names. * Move url prefix regexp to its own constant --- app/models/account.rb | 7 ++- app/models/concerns/account_interactions.rb | 9 ++- app/workers/activitypub/delivery_worker.rb | 6 +- .../concerns/account_interactions_spec.rb | 63 +++++++++++-------- .../activitypub/delivery_worker_spec.rb | 2 +- 5 files changed, 49 insertions(+), 38 deletions(-) diff --git a/app/models/account.rb b/app/models/account.rb index e6cf03fa8..9069b292f 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -55,8 +55,9 @@ # class Account < ApplicationRecord - USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i - MENTION_RE = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[a-z0-9]+)?)/i + USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i + MENTION_RE = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[a-z0-9]+)?)/i + URL_PREFIX_RE = /\Ahttp(s?):\/\/[^\/]+/ include AccountAssociations include AccountAvatar @@ -381,7 +382,7 @@ class Account < ApplicationRecord def synchronization_uri_prefix return 'local' if local? - @synchronization_uri_prefix ||= uri[/http(s?):\/\/[^\/]+\//] + @synchronization_uri_prefix ||= "#{uri[URL_PREFIX_RE]}/" end class Field < ActiveModelSerializers::Model diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb index 974f57820..b6439a53e 100644 --- a/app/models/concerns/account_interactions.rb +++ b/app/models/concerns/account_interactions.rb @@ -243,10 +243,13 @@ module AccountInteractions .where('users.current_sign_in_at > ?', User::ACTIVE_DURATION.ago) end - def remote_followers_hash(url_prefix) - Rails.cache.fetch("followers_hash:#{id}:#{url_prefix}") do + def remote_followers_hash(url) + url_prefix = url[Account::URL_PREFIX_RE] + return if url_prefix.blank? + + Rails.cache.fetch("followers_hash:#{id}:#{url_prefix}/") do digest = "\x00" * 32 - followers.where(Account.arel_table[:uri].matches(url_prefix + '%', false, true)).pluck_each(:uri) do |uri| + followers.where(Account.arel_table[:uri].matches("#{Account.sanitize_sql_like(url_prefix)}/%", false, true)).or(followers.where(uri: url_prefix)).pluck_each(:uri) do |uri| Xorcist.xor!(digest, Digest::SHA256.digest(uri)) end digest.unpack('H*')[0] diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb index 6c5a576a7..788f2cf80 100644 --- a/app/workers/activitypub/delivery_worker.rb +++ b/app/workers/activitypub/delivery_worker.rb @@ -44,11 +44,7 @@ class ActivityPub::DeliveryWorker end def synchronization_header - "collectionId=\"#{account_followers_url(@source_account)}\", digest=\"#{@source_account.remote_followers_hash(inbox_url_prefix)}\", url=\"#{account_followers_synchronization_url(@source_account)}\"" - end - - def inbox_url_prefix - @inbox_url[/http(s?):\/\/[^\/]+\//] + "collectionId=\"#{account_followers_url(@source_account)}\", digest=\"#{@source_account.remote_followers_hash(@inbox_url)}\", url=\"#{account_followers_synchronization_url(@source_account)}\"" end def perform_request diff --git a/spec/models/concerns/account_interactions_spec.rb b/spec/models/concerns/account_interactions_spec.rb index 85fbf7e79..ca243ebc5 100644 --- a/spec/models/concerns/account_interactions_spec.rb +++ b/spec/models/concerns/account_interactions_spec.rb @@ -539,46 +539,57 @@ describe AccountInteractions do end end - describe '#followers_hash' do + describe '#remote_followers_hash' do let(:me) { Fabricate(:account, username: 'Me') } let(:remote_1) { Fabricate(:account, username: 'alice', domain: 'example.org', uri: 'https://example.org/users/alice') } let(:remote_2) { Fabricate(:account, username: 'bob', domain: 'example.org', uri: 'https://example.org/users/bob') } - let(:remote_3) { Fabricate(:account, username: 'eve', domain: 'foo.org', uri: 'https://foo.org/users/eve') } + let(:remote_3) { Fabricate(:account, username: 'instance-actor', domain: 'example.org', uri: 'https://example.org') } + let(:remote_4) { Fabricate(:account, username: 'eve', domain: 'foo.org', uri: 'https://foo.org/users/eve') } before do remote_1.follow!(me) remote_2.follow!(me) remote_3.follow!(me) + remote_4.follow!(me) me.follow!(remote_1) end - context 'on a local user' do - it 'returns correct hash for remote domains' do - expect(me.remote_followers_hash('https://example.org/')).to eq '707962e297b7bd94468a21bc8e506a1bcea607a9142cd64e27c9b106b2a5f6ec' - expect(me.remote_followers_hash('https://foo.org/')).to eq 'ccb9c18a67134cfff9d62c7f7e7eb88e6b803446c244b84265565f4eba29df0e' - end - - it 'invalidates cache as needed when removing or adding followers' do - expect(me.remote_followers_hash('https://example.org/')).to eq '707962e297b7bd94468a21bc8e506a1bcea607a9142cd64e27c9b106b2a5f6ec' - remote_1.unfollow!(me) - expect(me.remote_followers_hash('https://example.org/')).to eq '241b00794ce9b46aa864f3220afadef128318da2659782985bac5ed5bd436bff' - remote_1.follow!(me) - expect(me.remote_followers_hash('https://example.org/')).to eq '707962e297b7bd94468a21bc8e506a1bcea607a9142cd64e27c9b106b2a5f6ec' - end + it 'returns correct hash for remote domains' do + expect(me.remote_followers_hash('https://example.org/')).to eq '20aecbe774b3d61c25094370baf370012b9271c5b172ecedb05caff8d79ef0c7' + expect(me.remote_followers_hash('https://foo.org/')).to eq 'ccb9c18a67134cfff9d62c7f7e7eb88e6b803446c244b84265565f4eba29df0e' + expect(me.remote_followers_hash('https://foo.org.evil.com/')).to eq '0000000000000000000000000000000000000000000000000000000000000000' + expect(me.remote_followers_hash('https://foo')).to eq '0000000000000000000000000000000000000000000000000000000000000000' end - context 'on a remote user' do - it 'returns correct hash for remote domains' do - expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me)) - end + it 'invalidates cache as needed when removing or adding followers' do + expect(me.remote_followers_hash('https://example.org/')).to eq '20aecbe774b3d61c25094370baf370012b9271c5b172ecedb05caff8d79ef0c7' + remote_3.unfollow!(me) + expect(me.remote_followers_hash('https://example.org/')).to eq '707962e297b7bd94468a21bc8e506a1bcea607a9142cd64e27c9b106b2a5f6ec' + remote_1.unfollow!(me) + expect(me.remote_followers_hash('https://example.org/')).to eq '241b00794ce9b46aa864f3220afadef128318da2659782985bac5ed5bd436bff' + remote_1.follow!(me) + expect(me.remote_followers_hash('https://example.org/')).to eq '707962e297b7bd94468a21bc8e506a1bcea607a9142cd64e27c9b106b2a5f6ec' + end + end - it 'invalidates cache as needed when removing or adding followers' do - expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me)) - me.unfollow!(remote_1) - expect(remote_1.local_followers_hash).to eq '0000000000000000000000000000000000000000000000000000000000000000' - me.follow!(remote_1) - expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me)) - end + describe '#local_followers_hash' do + let(:me) { Fabricate(:account, username: 'Me') } + let(:remote_1) { Fabricate(:account, username: 'alice', domain: 'example.org', uri: 'https://example.org/users/alice') } + + before do + me.follow!(remote_1) + end + + it 'returns correct hash for local users' do + expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me)) + end + + it 'invalidates cache as needed when removing or adding followers' do + expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me)) + me.unfollow!(remote_1) + expect(remote_1.local_followers_hash).to eq '0000000000000000000000000000000000000000000000000000000000000000' + me.follow!(remote_1) + expect(remote_1.local_followers_hash).to eq Digest::SHA256.hexdigest(ActivityPub::TagManager.instance.uri_for(me)) end end diff --git a/spec/workers/activitypub/delivery_worker_spec.rb b/spec/workers/activitypub/delivery_worker_spec.rb index f4633731e..d39393d50 100644 --- a/spec/workers/activitypub/delivery_worker_spec.rb +++ b/spec/workers/activitypub/delivery_worker_spec.rb @@ -11,7 +11,7 @@ describe ActivityPub::DeliveryWorker do let(:payload) { 'test' } before do - allow_any_instance_of(Account).to receive(:remote_followers_hash).with('https://example.com/').and_return('somehash') + allow_any_instance_of(Account).to receive(:remote_followers_hash).with('https://example.com/api').and_return('somehash') end describe 'perform' do From 19edb7a3f4e81b1f08fc0d7851e31f0e9f558520 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 15 Sep 2021 18:51:16 +0200 Subject: [PATCH 41/70] Fix followers synchronization mechanism not working when URI has empty path (#16744) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #16510, forgot the controller exposing the actual followers… --- .../activitypub/followers_synchronizations_controller.rb | 4 ++-- .../activitypub/followers_synchronizations_controller_spec.rb | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/controllers/activitypub/followers_synchronizations_controller.rb b/app/controllers/activitypub/followers_synchronizations_controller.rb index 525031105..940b77cf0 100644 --- a/app/controllers/activitypub/followers_synchronizations_controller.rb +++ b/app/controllers/activitypub/followers_synchronizations_controller.rb @@ -19,11 +19,11 @@ class ActivityPub::FollowersSynchronizationsController < ActivityPub::BaseContro private def uri_prefix - signed_request_account.uri[/http(s?):\/\/[^\/]+\//] + signed_request_account.uri[Account::URL_PREFIX_RE] end def set_items - @items = @account.followers.where(Account.arel_table[:uri].matches(uri_prefix + '%', false, true)).pluck(:uri) + @items = @account.followers.where(Account.arel_table[:uri].matches("#{Account.sanitize_sql_like(uri_prefix)}/%", false, true)).or(@account.followers.where(uri: uri_prefix)).pluck(:uri) end def collection_presenter diff --git a/spec/controllers/activitypub/followers_synchronizations_controller_spec.rb b/spec/controllers/activitypub/followers_synchronizations_controller_spec.rb index 88f4554c2..ca89dc2ae 100644 --- a/spec/controllers/activitypub/followers_synchronizations_controller_spec.rb +++ b/spec/controllers/activitypub/followers_synchronizations_controller_spec.rb @@ -5,11 +5,13 @@ RSpec.describe ActivityPub::FollowersSynchronizationsController, type: :controll let!(:follower_1) { Fabricate(:account, domain: 'example.com', uri: 'https://example.com/users/a') } let!(:follower_2) { Fabricate(:account, domain: 'example.com', uri: 'https://example.com/users/b') } let!(:follower_3) { Fabricate(:account, domain: 'foo.com', uri: 'https://foo.com/users/a') } + let!(:follower_4) { Fabricate(:account, username: 'instance-actor', domain: 'example.com', uri: 'https://example.com') } before do follower_1.follow!(account) follower_2.follow!(account) follower_3.follow!(account) + follower_4.follow!(account) end before do @@ -45,7 +47,7 @@ RSpec.describe ActivityPub::FollowersSynchronizationsController, type: :controll it 'returns orderedItems with followers from example.com' do expect(body[:orderedItems]).to be_an Array - expect(body[:orderedItems].sort).to eq [follower_1.uri, follower_2.uri] + expect(body[:orderedItems].sort).to eq [follower_4.uri, follower_1.uri, follower_2.uri] end it 'returns private Cache-Control header' do From c0b2c2c166348ab98b59f74b1a5c001e2401da5e Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 21 Jun 2021 20:14:47 +0200 Subject: [PATCH 42/70] Fix serialization of followers/following counts when user hides their network (#16418) * Add tests * Fix serialization of followers/following counts when user hides their network Fixes #16382 Signed-off-by: Claire --- .../follower_accounts_controller.rb | 2 +- .../following_accounts_controller.rb | 2 +- .../follower_accounts_controller_spec.rb | 19 ++++++++++++++++++- .../following_accounts_controller_spec.rb | 19 ++++++++++++++++++- 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/app/controllers/follower_accounts_controller.rb b/app/controllers/follower_accounts_controller.rb index ff4df2adf..b3589a39f 100644 --- a/app/controllers/follower_accounts_controller.rb +++ b/app/controllers/follower_accounts_controller.rb @@ -85,7 +85,7 @@ class FollowerAccountsController < ApplicationController if page_requested? || !@account.user_hides_network? # Return all fields else - %i(id type totalItems) + %i(id type total_items) end end end diff --git a/app/controllers/following_accounts_controller.rb b/app/controllers/following_accounts_controller.rb index 6bb95c454..8a72dc475 100644 --- a/app/controllers/following_accounts_controller.rb +++ b/app/controllers/following_accounts_controller.rb @@ -85,7 +85,7 @@ class FollowingAccountsController < ApplicationController if page_requested? || !@account.user_hides_network? # Return all fields else - %i(id type totalItems) + %i(id type total_items) end end end diff --git a/spec/controllers/follower_accounts_controller_spec.rb b/spec/controllers/follower_accounts_controller_spec.rb index f6d55f693..006274169 100644 --- a/spec/controllers/follower_accounts_controller_spec.rb +++ b/spec/controllers/follower_accounts_controller_spec.rb @@ -3,7 +3,7 @@ require 'rails_helper' describe FollowerAccountsController do render_views - let(:alice) { Fabricate(:account, username: 'alice') } + let(:alice) { Fabricate(:user).account } let(:follower0) { Fabricate(:account) } let(:follower1) { Fabricate(:account) } @@ -101,6 +101,23 @@ describe FollowerAccountsController do expect(body['partOf']).to be_blank end + context 'when account hides their network' do + before do + alice.user.settings.hide_network = true + end + + it 'returns followers count' do + expect(body['totalItems']).to eq 2 + end + + it 'does not return items' do + expect(body['items']).to be_blank + expect(body['orderedItems']).to be_blank + expect(body['first']).to be_blank + expect(body['last']).to be_blank + end + end + context 'when account is permanently suspended' do before do alice.suspend! diff --git a/spec/controllers/following_accounts_controller_spec.rb b/spec/controllers/following_accounts_controller_spec.rb index 0fc0967a6..7ec0e3d06 100644 --- a/spec/controllers/following_accounts_controller_spec.rb +++ b/spec/controllers/following_accounts_controller_spec.rb @@ -3,7 +3,7 @@ require 'rails_helper' describe FollowingAccountsController do render_views - let(:alice) { Fabricate(:account, username: 'alice') } + let(:alice) { Fabricate(:user).account } let(:followee0) { Fabricate(:account) } let(:followee1) { Fabricate(:account) } @@ -101,6 +101,23 @@ describe FollowingAccountsController do expect(body['partOf']).to be_blank end + context 'when account hides their network' do + before do + alice.user.settings.hide_network = true + end + + it 'returns followers count' do + expect(body['totalItems']).to eq 2 + end + + it 'does not return items' do + expect(body['items']).to be_blank + expect(body['orderedItems']).to be_blank + expect(body['first']).to be_blank + expect(body['last']).to be_blank + end + end + context 'when account is permanently suspended' do before do alice.suspend! From 6c344d90c7b0a4aa7afbe4d46d1e98bbd341ef4e Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 14 Oct 2021 19:59:28 +0200 Subject: [PATCH 43/70] Fix tootctl self-destruct not sending Delete activities for recently-suspended accounts (#16688) * Do not block existing users' emails on self-destruct That is wasteful and unintuitive * Do not close registrations when running tootctl self-destruct with --dry-run * Close registrations on self-destruct regardless of known remote accounts * Fix tootctl self-destruct not sending Deletes for recently-suspended accounts * Suspend local users even if no remote account is known * Do not show scary confirmation text if ran with --dry-run --- lib/cli.rb | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/lib/cli.rb b/lib/cli.rb index 3f1658566..8815e137a 100644 --- a/lib/cli.rb +++ b/lib/cli.rb @@ -94,17 +94,22 @@ module Mastodon exit(1) unless prompt.ask('Type in the domain of the server to confirm:', required: true) == Rails.configuration.x.local_domain - prompt.warn('This operation WILL NOT be reversible. It can also take a long time.') - prompt.warn('While the data won\'t be erased locally, the server will be in a BROKEN STATE afterwards.') - prompt.warn('A running Sidekiq process is required. Do not shut it down until queues clear.') + unless options[:dry_run] + prompt.warn('This operation WILL NOT be reversible. It can also take a long time.') + prompt.warn('While the data won\'t be erased locally, the server will be in a BROKEN STATE afterwards.') + prompt.warn('A running Sidekiq process is required. Do not shut it down until queues clear.') - exit(1) if prompt.no?('Are you sure you want to proceed?') + exit(1) if prompt.no?('Are you sure you want to proceed?') + end inboxes = Account.inboxes processed = 0 dry_run = options[:dry_run] ? ' (DRY RUN)' : '' + Setting.registrations_mode = 'none' unless options[:dry_run] + if inboxes.empty? + Account.local.without_suspended.in_batches.update_all(suspended_at: Time.now.utc, suspension_origin: :local) unless options[:dry_run] prompt.ok('It seems like your server has not federated with anything') prompt.ok('You can shut it down and delete it any time') return @@ -112,9 +117,7 @@ module Mastodon prompt.warn('Do NOT interrupt this process...') - Setting.registrations_mode = 'none' - - Account.local.without_suspended.find_each do |account| + delete_account = ->(account) do payload = ActiveModelSerializers::SerializableResource.new( account, serializer: ActivityPub::DeleteActorSerializer, @@ -128,12 +131,15 @@ module Mastodon [json, account.id, inbox_url] end - account.suspend! + account.suspend!(block_email: false) end processed += 1 end + Account.local.without_suspended.find_each { |account| delete_account.call(account) } + Account.local.suspended.joins(:deletion_request).find_each { |account| delete_account.call(account) } + prompt.ok("Queued #{inboxes.size * processed} items into Sidekiq for #{processed} accounts#{dry_run}") prompt.ok('Wait until Sidekiq processes all items, then you can shut everything down and delete the data') rescue TTY::Reader::InputInterrupt From d2cdfe92ed54cfe0110a8734bc9d6f59c24ba2e5 Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 25 Oct 2021 16:34:15 +0200 Subject: [PATCH 44/70] Fix mastodon:setup to take dotenv/docker-compose differences into account (#16896) In order to work around https://github.com/mastodon/mastodon/issues/16895, add a warning to .env.production.sample, and change the mastodon:setup rake task to: - output a warning if a variable will be interpreted differently by dotenv and docker-compose - ensure the printed config is compatible with docker-compose --- .env.production.sample | 6 ++++++ lib/tasks/mastodon.rake | 15 +++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.env.production.sample b/.env.production.sample index 6f14c5804..ff6db83ca 100644 --- a/.env.production.sample +++ b/.env.production.sample @@ -4,6 +4,12 @@ # not demonstrate all available configuration options. Please look at # https://docs.joinmastodon.org/admin/config/ for the full documentation. +# Note that this file accepts slightly different syntax depending on whether +# you are using `docker-compose` or not. In particular, if you use +# `docker-compose`, the value of each declared variable will be taken verbatim, +# including surrounding quotes. +# See: https://github.com/mastodon/mastodon/issues/16895 + # Federation # ---------- # This identifies your server and cannot be changed safely later diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index 2ad1e778b..2ea22a1b4 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -333,8 +333,12 @@ namespace :mastodon do prompt.say 'This configuration will be written to .env.production' if prompt.yes?('Save configuration?') + incompatible_syntax = false + env_contents = env.each_pair.map do |key, value| if value.is_a?(String) && value =~ /[\s\#\\"]/ + incompatible_syntax = true + if value =~ /[']/ value = value.to_s.gsub(/[\\"\$]/) { |x| "\\#{x}" } "#{key}=\"#{value}\"" @@ -346,12 +350,19 @@ namespace :mastodon do end end.join("\n") - File.write(Rails.root.join('.env.production'), "# Generated with mastodon:setup on #{Time.now.utc}\n\n" + env_contents + "\n") + generated_header = "# Generated with mastodon:setup on #{Time.now.utc}\n\n" + + if incompatible_syntax + generated_header << "Some variables in this file will be interpreted differently whether you are\n" + generated_header << "using docker-compose or not.\n\n" + end + + File.write(Rails.root.join('.env.production'), "#{generated_header}#{env_contents}\n") if using_docker prompt.ok 'Below is your configuration, save it to an .env.production file outside Docker:' prompt.say "\n" - prompt.say File.read(Rails.root.join('.env.production')) + prompt.say "#{generated_header}#{env.each_pair.map { |key, value| "#{key}=#{value}" }.join("\n")}" prompt.say "\n" prompt.ok 'It is also saved within this container so you can proceed with this wizard.' end From 0994c4b11a091ff474c3c1bc0dfe87226ef989da Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 11 Nov 2021 14:00:30 +0100 Subject: [PATCH 45/70] Fix "bundle exec rails mastodon:setup" crashing in some circumstances (#16976) Fix regression from #16896 --- lib/tasks/mastodon.rake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index 2ea22a1b4..c89c4726e 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -350,11 +350,11 @@ namespace :mastodon do end end.join("\n") - generated_header = "# Generated with mastodon:setup on #{Time.now.utc}\n\n" + generated_header = "# Generated with mastodon:setup on #{Time.now.utc}\n\n".dup if incompatible_syntax - generated_header << "Some variables in this file will be interpreted differently whether you are\n" - generated_header << "using docker-compose or not.\n\n" + generated_header << "# Some variables in this file will be interpreted differently whether you are\n" + generated_header << "# using docker-compose or not.\n\n" end File.write(Rails.root.join('.env.production'), "#{generated_header}#{env_contents}\n") From 5b07f4e90ebb884f82a9dc379365aa316f38167c Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 21 Oct 2021 20:39:35 +0200 Subject: [PATCH 46/70] Fix some link previews being incorrectly generated from other prior links (#16885) * Add tests * Fix some link previews being incorrectly generated from different prior links PR #12403 added a cache to avoid redundant queries when the OEmbed endpoint can be guessed from the URL. This caching mechanism is not perfectly correct as there is no guarantee that all pages from a given domain share the same OEmbed provider endpoint. This PR prevents the FetchOEmbedService from caching OEmbed endpoint that cannot be generalized by replacing a fully-qualified URL from the endpoint's parameters, greatly reducing the number of incorrect cached generalizations. --- app/services/fetch_oembed_service.rb | 5 ++- spec/fixtures/requests/oembed_youtube.html | 7 ++++ spec/services/fetch_oembed_service_spec.rb | 41 ++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 spec/fixtures/requests/oembed_youtube.html diff --git a/app/services/fetch_oembed_service.rb b/app/services/fetch_oembed_service.rb index 67e33875c..38ad3b989 100644 --- a/app/services/fetch_oembed_service.rb +++ b/app/services/fetch_oembed_service.rb @@ -2,6 +2,7 @@ class FetchOEmbedService ENDPOINT_CACHE_EXPIRES_IN = 24.hours.freeze + URL_REGEX = /(=(http[s]?(%3A|:)(\/\/|%2F%2F)))([^&]*)/i.freeze attr_reader :url, :options, :format, :endpoint_url @@ -55,10 +56,12 @@ class FetchOEmbedService end def cache_endpoint! + return unless URL_REGEX.match?(@endpoint_url) + url_domain = Addressable::URI.parse(@url).normalized_host endpoint_hash = { - endpoint: @endpoint_url.gsub(/(=(http[s]?(%3A|:)(\/\/|%2F%2F)))([^&]*)/i, '={url}'), + endpoint: @endpoint_url.gsub(URL_REGEX, '={url}'), format: @format, } diff --git a/spec/fixtures/requests/oembed_youtube.html b/spec/fixtures/requests/oembed_youtube.html new file mode 100644 index 000000000..1508e4dd9 --- /dev/null +++ b/spec/fixtures/requests/oembed_youtube.html @@ -0,0 +1,7 @@ + + + + + + + diff --git a/spec/services/fetch_oembed_service_spec.rb b/spec/services/fetch_oembed_service_spec.rb index a4262b040..88f0113ed 100644 --- a/spec/services/fetch_oembed_service_spec.rb +++ b/spec/services/fetch_oembed_service_spec.rb @@ -13,6 +13,32 @@ describe FetchOEmbedService, type: :service do describe 'discover_provider' do context 'when status code is 200 and MIME type is text/html' do + context 'when OEmbed endpoint contains URL as parameter' do + before do + stub_request(:get, 'https://www.youtube.com/watch?v=IPSbNdBmWKE').to_return( + status: 200, + headers: { 'Content-Type': 'text/html' }, + body: request_fixture('oembed_youtube.html'), + ) + stub_request(:get, 'https://www.youtube.com/oembed?format=json&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DIPSbNdBmWKE').to_return( + status: 200, + headers: { 'Content-Type': 'text/html' }, + body: request_fixture('oembed_json_empty.html') + ) + end + + it 'returns new OEmbed::Provider for JSON provider' do + subject.call('https://www.youtube.com/watch?v=IPSbNdBmWKE') + expect(subject.endpoint_url).to eq 'https://www.youtube.com/oembed?format=json&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DIPSbNdBmWKE' + expect(subject.format).to eq :json + end + + it 'stores URL template' do + subject.call('https://www.youtube.com/watch?v=IPSbNdBmWKE') + expect(Rails.cache.read('oembed_endpoint:www.youtube.com')[:endpoint]).to eq 'https://www.youtube.com/oembed?format=json&url={url}' + end + end + context 'Both of JSON and XML provider are discoverable' do before do stub_request(:get, 'https://host.test/oembed.html').to_return( @@ -33,6 +59,11 @@ describe FetchOEmbedService, type: :service do expect(subject.endpoint_url).to eq 'https://host.test/provider.xml' expect(subject.format).to eq :xml end + + it 'does not cache OEmbed endpoint' do + subject.call('https://host.test/oembed.html', format: :xml) + expect(Rails.cache.exist?('oembed_endpoint:host.test')).to eq false + end end context 'JSON provider is discoverable while XML provider is not' do @@ -49,6 +80,11 @@ describe FetchOEmbedService, type: :service do expect(subject.endpoint_url).to eq 'https://host.test/provider.json' expect(subject.format).to eq :json end + + it 'does not cache OEmbed endpoint' do + subject.call('https://host.test/oembed.html') + expect(Rails.cache.exist?('oembed_endpoint:host.test')).to eq false + end end context 'XML provider is discoverable while JSON provider is not' do @@ -65,6 +101,11 @@ describe FetchOEmbedService, type: :service do expect(subject.endpoint_url).to eq 'https://host.test/provider.xml' expect(subject.format).to eq :xml end + + it 'does not cache OEmbed endpoint' do + subject.call('https://host.test/oembed.html') + expect(Rails.cache.exist?('oembed_endpoint:host.test')).to eq false + end end context 'Invalid XML provider is discoverable while JSON provider is not' do From d33b3610002d7e6c6fe8341a43622064b7cf9131 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 8 Sep 2021 23:33:23 +0200 Subject: [PATCH 47/70] Fix addressing of remote groups' followers (#16700) Fixes #16699 --- app/lib/activitypub/tag_manager.rb | 26 ++++++++++++++---------- spec/lib/activitypub/tag_manager_spec.rb | 8 ++++++++ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb index f6b5e10d3..f6b9741fa 100644 --- a/app/lib/activitypub/tag_manager.rb +++ b/app/lib/activitypub/tag_manager.rb @@ -64,6 +64,10 @@ class ActivityPub::TagManager account_status_replies_url(target.account, target, page_params) end + def followers_uri_for(target) + target.local? ? account_followers_url(target) : target.followers_url.presence + end + # Primary audience of a status # Public statuses go out to primarily the public collection # Unlisted and private statuses go out primarily to the followers collection @@ -80,17 +84,17 @@ class ActivityPub::TagManager account_ids = status.active_mentions.pluck(:account_id) to = status.account.followers.where(id: account_ids).each_with_object([]) do |account, result| result << uri_for(account) - result << account_followers_url(account) if account.group? + result << followers_uri_for(account) if account.group? end to.concat(FollowRequest.where(target_account_id: status.account_id, account_id: account_ids).each_with_object([]) do |request, result| result << uri_for(request.account) - result << account_followers_url(request.account) if request.account.group? - end) + result << followers_uri_for(request.account) if request.account.group? + end).compact else status.active_mentions.each_with_object([]) do |mention, result| result << uri_for(mention.account) - result << account_followers_url(mention.account) if mention.account.group? - end + result << followers_uri_for(mention.account) if mention.account.group? + end.compact end end end @@ -118,17 +122,17 @@ class ActivityPub::TagManager account_ids = status.active_mentions.pluck(:account_id) cc.concat(status.account.followers.where(id: account_ids).each_with_object([]) do |account, result| result << uri_for(account) - result << account_followers_url(account) if account.group? - end) + result << followers_uri_for(account) if account.group? + end.compact) cc.concat(FollowRequest.where(target_account_id: status.account_id, account_id: account_ids).each_with_object([]) do |request, result| result << uri_for(request.account) - result << account_followers_url(request.account) if request.account.group? - end) + result << followers_uri_for(request.account) if request.account.group? + end.compact) else cc.concat(status.active_mentions.each_with_object([]) do |mention, result| result << uri_for(mention.account) - result << account_followers_url(mention.account) if mention.account.group? - end) + result << followers_uri_for(mention.account) if mention.account.group? + end.compact) end end diff --git a/spec/lib/activitypub/tag_manager_spec.rb b/spec/lib/activitypub/tag_manager_spec.rb index 1c5c6f0ed..606a1de2e 100644 --- a/spec/lib/activitypub/tag_manager_spec.rb +++ b/spec/lib/activitypub/tag_manager_spec.rb @@ -42,6 +42,14 @@ RSpec.describe ActivityPub::TagManager do expect(subject.to(status)).to eq [subject.uri_for(mentioned)] end + it "returns URIs of mentioned group's followers for direct statuses to groups" do + status = Fabricate(:status, visibility: :direct) + mentioned = Fabricate(:account, domain: 'remote.org', uri: 'https://remote.org/group', followers_url: 'https://remote.org/group/followers', actor_type: 'Group') + status.mentions.create(account: mentioned) + expect(subject.to(status)).to include(subject.uri_for(mentioned)) + expect(subject.to(status)).to include(subject.followers_uri_for(mentioned)) + end + it "returns URIs of mentions for direct silenced author's status only if they are followers or requesting to be" do bob = Fabricate(:account, username: 'bob') alice = Fabricate(:account, username: 'alice') From e10920eb20e105ef3ee4cd81afe4c8a9f0039f19 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 1 Sep 2021 22:06:40 +0200 Subject: [PATCH 48/70] Fix processing mentions to domains with non-ascii TLDs (#16689) Fixes #16602 --- app/models/account.rb | 2 +- spec/services/process_mentions_service_spec.rb | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/app/models/account.rb b/app/models/account.rb index 9069b292f..007adc409 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -56,7 +56,7 @@ class Account < ApplicationRecord USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i - MENTION_RE = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[a-z0-9]+)?)/i + MENTION_RE = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[[:word:]]+)?)/i URL_PREFIX_RE = /\Ahttp(s?):\/\/[^\/]+/ include AccountAssociations diff --git a/spec/services/process_mentions_service_spec.rb b/spec/services/process_mentions_service_spec.rb index c30de8eeb..69f8310c2 100644 --- a/spec/services/process_mentions_service_spec.rb +++ b/spec/services/process_mentions_service_spec.rb @@ -73,6 +73,24 @@ RSpec.describe ProcessMentionsService, type: :service do expect(a_request(:post, remote_user.inbox_url)).to have_been_made.once end end + + context 'with an IDN TLD' do + let(:remote_user) { Fabricate(:account, username: 'foo', protocol: :activitypub, domain: 'xn--y9a3aq.xn--y9a3aq', inbox_url: 'http://example.com/inbox') } + let(:status) { Fabricate(:status, account: account, text: "Hello @foo@հայ.հայ") } + + before do + stub_request(:post, remote_user.inbox_url) + subject.call(status) + end + + it 'creates a mention' do + expect(remote_user.mentions.where(status: status).count).to eq 1 + end + + it 'sends activity to the inbox' do + expect(a_request(:post, remote_user.inbox_url)).to have_been_made.once + end + end end context 'Temporarily-unreachable ActivityPub user' do From edc55002cfa394dc9d3cc0962cbaa730732f7187 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 25 Aug 2021 22:52:41 +0200 Subject: [PATCH 49/70] Fix authentication failures after going halfway through a sign-in attempt (#16607) * Add tests * Add security-related tests My first (unpublished) attempt at fixing the issues introduced (extremely hard-to-exploit) security vulnerabilities, addressing them in a test. * Fix authentication failures after going halfway through a sign-in attempt * Refactor `authenticate_with_sign_in_token` and `authenticate_with_two_factor` to make the two authentication steps more obvious --- app/controllers/auth/sessions_controller.rb | 16 ++- .../sign_in_token_authentication_concern.rb | 18 +-- .../two_factor_authentication_concern.rb | 22 ++-- .../auth/sessions_controller_spec.rb | 109 ++++++++++++++++++ 4 files changed, 143 insertions(+), 22 deletions(-) diff --git a/app/controllers/auth/sessions_controller.rb b/app/controllers/auth/sessions_controller.rb index 13d158c67..d2bd6c4a0 100644 --- a/app/controllers/auth/sessions_controller.rb +++ b/app/controllers/auth/sessions_controller.rb @@ -58,16 +58,20 @@ class Auth::SessionsController < Devise::SessionsController protected def find_user - if session[:attempt_user_id] + if user_params[:email].present? + find_user_from_params + elsif session[:attempt_user_id] User.find_by(id: session[:attempt_user_id]) - else - user = User.authenticate_with_ldap(user_params) if Devise.ldap_authentication - user ||= User.authenticate_with_pam(user_params) if Devise.pam_authentication - user ||= User.find_for_authentication(email: user_params[:email]) - user end end + def find_user_from_params + user = User.authenticate_with_ldap(user_params) if Devise.ldap_authentication + user ||= User.authenticate_with_pam(user_params) if Devise.pam_authentication + user ||= User.find_for_authentication(email: user_params[:email]) + user + end + def user_params params.require(:user).permit(:email, :password, :otp_attempt, :sign_in_token_attempt, credential: {}) end diff --git a/app/controllers/concerns/sign_in_token_authentication_concern.rb b/app/controllers/concerns/sign_in_token_authentication_concern.rb index 3c95a4afd..33ea5cf85 100644 --- a/app/controllers/concerns/sign_in_token_authentication_concern.rb +++ b/app/controllers/concerns/sign_in_token_authentication_concern.rb @@ -16,14 +16,18 @@ module SignInTokenAuthenticationConcern end def authenticate_with_sign_in_token - user = self.resource = find_user + if user_params[:email].present? + user = self.resource = find_user_from_params + prompt_for_sign_in_token(user) if user&.external_or_valid_password?(user_params[:password]) + elsif session[:attempt_user_id] + user = self.resource = User.find_by(id: session[:attempt_user_id]) + return if user.nil? - if user.present? && session[:attempt_user_id].present? && session[:attempt_user_updated_at] != user.updated_at.to_s - restart_session - elsif user_params.key?(:sign_in_token_attempt) && session[:attempt_user_id] - authenticate_with_sign_in_token_attempt(user) - elsif user.present? && user.external_or_valid_password?(user_params[:password]) - prompt_for_sign_in_token(user) + if session[:attempt_user_updated_at] != user.updated_at.to_s + restart_session + elsif user_params.key?(:sign_in_token_attempt) + authenticate_with_sign_in_token_attempt(user) + end end end diff --git a/app/controllers/concerns/two_factor_authentication_concern.rb b/app/controllers/concerns/two_factor_authentication_concern.rb index 4d4ccf49c..402638eba 100644 --- a/app/controllers/concerns/two_factor_authentication_concern.rb +++ b/app/controllers/concerns/two_factor_authentication_concern.rb @@ -35,16 +35,20 @@ module TwoFactorAuthenticationConcern end def authenticate_with_two_factor - user = self.resource = find_user + if user_params[:email].present? + user = self.resource = find_user_from_params + prompt_for_two_factor(user) if user&.external_or_valid_password?(user_params[:password]) + elsif session[:attempt_user_id] + user = self.resource = User.find_by(id: session[:attempt_user_id]) + return if user.nil? - if user.present? && session[:attempt_user_id].present? && session[:attempt_user_updated_at] != user.updated_at.to_s - restart_session - elsif user.webauthn_enabled? && user_params.key?(:credential) && session[:attempt_user_id] - authenticate_with_two_factor_via_webauthn(user) - elsif user_params.key?(:otp_attempt) && session[:attempt_user_id] - authenticate_with_two_factor_via_otp(user) - elsif user.present? && user.external_or_valid_password?(user_params[:password]) - prompt_for_two_factor(user) + if session[:attempt_user_updated_at] != user.updated_at.to_s + restart_session + elsif user.webauthn_enabled? && user_params.key?(:credential) + authenticate_with_two_factor_via_webauthn(user) + elsif user_params.key?(:otp_attempt) + authenticate_with_two_factor_via_otp(user) + end end end diff --git a/spec/controllers/auth/sessions_controller_spec.rb b/spec/controllers/auth/sessions_controller_spec.rb index d3a9a11eb..8190ba107 100644 --- a/spec/controllers/auth/sessions_controller_spec.rb +++ b/spec/controllers/auth/sessions_controller_spec.rb @@ -206,6 +206,38 @@ RSpec.describe Auth::SessionsController, type: :controller do end end + context 'using email and password after an unfinished log-in attempt to a 2FA-protected account' do + let!(:other_user) do + Fabricate(:user, email: 'z@y.com', password: 'abcdefgh', otp_required_for_login: true, otp_secret: User.generate_otp_secret(32)) + end + + before do + post :create, params: { user: { email: other_user.email, password: other_user.password } } + post :create, params: { user: { email: user.email, password: user.password } } + end + + it 'renders two factor authentication page' do + expect(controller).to render_template("two_factor") + expect(controller).to render_template(partial: "_otp_authentication_form") + end + end + + context 'using email and password after an unfinished log-in attempt with a sign-in token challenge' do + let!(:other_user) do + Fabricate(:user, email: 'z@y.com', password: 'abcdefgh', otp_required_for_login: false, current_sign_in_at: 1.month.ago) + end + + before do + post :create, params: { user: { email: other_user.email, password: other_user.password } } + post :create, params: { user: { email: user.email, password: user.password } } + end + + it 'renders two factor authentication page' do + expect(controller).to render_template("two_factor") + expect(controller).to render_template(partial: "_otp_authentication_form") + end + end + context 'using upcase email and password' do before do post :create, params: { user: { email: user.email.upcase, password: user.password } } @@ -231,6 +263,21 @@ RSpec.describe Auth::SessionsController, type: :controller do end end + context 'using a valid OTP, attempting to leverage previous half-login to bypass password auth' do + let!(:other_user) do + Fabricate(:user, email: 'z@y.com', password: 'abcdefgh', otp_required_for_login: false, current_sign_in_at: 1.month.ago) + end + + before do + post :create, params: { user: { email: other_user.email, password: other_user.password } } + post :create, params: { user: { email: user.email, otp_attempt: user.current_otp } }, session: { attempt_user_updated_at: user.updated_at.to_s } + end + + it "doesn't log the user in" do + expect(controller.current_user).to be_nil + end + end + context 'when the server has an decryption error' do before do allow_any_instance_of(User).to receive(:validate_and_consume_otp!).and_raise(OpenSSL::Cipher::CipherError) @@ -380,6 +427,52 @@ RSpec.describe Auth::SessionsController, type: :controller do end end + context 'using email and password after an unfinished log-in attempt to a 2FA-protected account' do + let!(:other_user) do + Fabricate(:user, email: 'z@y.com', password: 'abcdefgh', otp_required_for_login: true, otp_secret: User.generate_otp_secret(32)) + end + + before do + post :create, params: { user: { email: other_user.email, password: other_user.password } } + post :create, params: { user: { email: user.email, password: user.password } } + end + + it 'renders sign in token authentication page' do + expect(controller).to render_template("sign_in_token") + end + + it 'generates sign in token' do + expect(user.reload.sign_in_token).to_not be_nil + end + + it 'sends sign in token e-mail' do + expect(UserMailer).to have_received(:sign_in_token) + end + end + + context 'using email and password after an unfinished log-in attempt with a sign-in token challenge' do + let!(:other_user) do + Fabricate(:user, email: 'z@y.com', password: 'abcdefgh', otp_required_for_login: false, current_sign_in_at: 1.month.ago) + end + + before do + post :create, params: { user: { email: other_user.email, password: other_user.password } } + post :create, params: { user: { email: user.email, password: user.password } } + end + + it 'renders sign in token authentication page' do + expect(controller).to render_template("sign_in_token") + end + + it 'generates sign in token' do + expect(user.reload.sign_in_token).to_not be_nil + end + + it 'sends sign in token e-mail' do + expect(UserMailer).to have_received(:sign_in_token).with(user, any_args) + end + end + context 'using a valid sign in token' do before do user.generate_sign_in_token && user.save @@ -395,6 +488,22 @@ RSpec.describe Auth::SessionsController, type: :controller do end end + context 'using a valid sign in token, attempting to leverage previous half-login to bypass password auth' do + let!(:other_user) do + Fabricate(:user, email: 'z@y.com', password: 'abcdefgh', otp_required_for_login: false, current_sign_in_at: 1.month.ago) + end + + before do + user.generate_sign_in_token && user.save + post :create, params: { user: { email: other_user.email, password: other_user.password } } + post :create, params: { user: { email: user.email, sign_in_token_attempt: user.sign_in_token } }, session: { attempt_user_updated_at: user.updated_at.to_s } + end + + it "doesn't log the user in" do + expect(controller.current_user).to be_nil + end + end + context 'using an invalid sign in token' do before do post :create, params: { user: { sign_in_token_attempt: 'wrongotp' } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s } From 2c02cb59eff7ded176f97cc54578b99f42d659a1 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 30 Sep 2021 05:26:29 +0200 Subject: [PATCH 50/70] Fix webauthn secure key authentication (#16792) * Add tests * Fix webauthn secure key authentication Fixes #16769 --- app/controllers/auth/sessions_controller.rb | 2 +- .../auth/sessions_controller_spec.rb | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/app/controllers/auth/sessions_controller.rb b/app/controllers/auth/sessions_controller.rb index d2bd6c4a0..ecb5a0a03 100644 --- a/app/controllers/auth/sessions_controller.rb +++ b/app/controllers/auth/sessions_controller.rb @@ -40,7 +40,7 @@ class Auth::SessionsController < Devise::SessionsController end def webauthn_options - user = find_user + user = User.find_by(id: session[:attempt_user_id]) if user.webauthn_enabled? options_for_get = WebAuthn::Credential.options_for_get( diff --git a/spec/controllers/auth/sessions_controller_spec.rb b/spec/controllers/auth/sessions_controller_spec.rb index 8190ba107..608859f8c 100644 --- a/spec/controllers/auth/sessions_controller_spec.rb +++ b/spec/controllers/auth/sessions_controller_spec.rb @@ -519,4 +519,33 @@ RSpec.describe Auth::SessionsController, type: :controller do end end end + + describe 'GET #webauthn_options' do + context 'with WebAuthn and OTP enabled as second factor' do + let(:domain) { "#{Rails.configuration.x.use_https ? 'https' : 'http' }://#{Rails.configuration.x.web_domain}" } + + let(:fake_client) { WebAuthn::FakeClient.new(domain) } + + let!(:user) do + Fabricate(:user, email: 'x@y.com', password: 'abcdefgh', otp_required_for_login: true, otp_secret: User.generate_otp_secret(32)) + end + + before do + user.update(webauthn_id: WebAuthn.generate_user_id) + public_key_credential = WebAuthn::Credential.from_create(fake_client.create) + user.webauthn_credentials.create( + nickname: 'SecurityKeyNickname', + external_id: public_key_credential.id, + public_key: public_key_credential.public_key, + sign_count: '1000' + ) + post :create, params: { user: { email: user.email, password: user.password } } + end + + it 'returns http success' do + get :webauthn_options + expect(response).to have_http_status :ok + end + end + end end From 168272fe610f9b43b0efff9d5d4a1860e466a387 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 20 Aug 2021 08:40:33 +0200 Subject: [PATCH 51/70] Fix remotely-suspended accounts' toots being merged back into timelines (#16628) * Fix remotely-suspended accounts' toots being merged back into timelines * Mark remotely-deleted accounts as remotely suspended --- app/services/resolve_account_service.rb | 1 + app/services/unsuspend_account_service.rb | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb index c78acaea8..de7aabc66 100644 --- a/app/services/resolve_account_service.rb +++ b/app/services/resolve_account_service.rb @@ -141,6 +141,7 @@ class ResolveAccountService < BaseService end def queue_deletion! + @account.suspend!(origin: :remote) AccountDeletionWorker.perform_async(@account.id, reserve_username: false, skip_activitypub: true) end diff --git a/app/services/unsuspend_account_service.rb b/app/services/unsuspend_account_service.rb index 949c670aa..7e52a7594 100644 --- a/app/services/unsuspend_account_service.rb +++ b/app/services/unsuspend_account_service.rb @@ -7,7 +7,7 @@ class UnsuspendAccountService < BaseService unsuspend! refresh_remote_account! - return if @account.nil? + return if @account.nil? || @account.suspended? merge_into_home_timelines! merge_into_list_timelines! From 80ca4fdb3c6fb23d20213b075ccf59106e0e3129 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 11 Aug 2021 16:40:55 +0200 Subject: [PATCH 52/70] Fix crash when encountering invalid account fields (#16598) * Add test * Fix crash when encountering invalid account fields --- app/models/account.rb | 6 +++++- spec/services/activitypub/process_account_service_spec.rb | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app/models/account.rb b/app/models/account.rb index 007adc409..089727529 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -302,7 +302,11 @@ class Account < ApplicationRecord end def fields - (self[:fields] || []).map { |f| Field.new(self, f) } + (self[:fields] || []).map do |f| + Field.new(self, f) + rescue + nil + end.compact end def fields_attributes=(attributes) diff --git a/spec/services/activitypub/process_account_service_spec.rb b/spec/services/activitypub/process_account_service_spec.rb index 56e7f8321..1b1d878a7 100644 --- a/spec/services/activitypub/process_account_service_spec.rb +++ b/spec/services/activitypub/process_account_service_spec.rb @@ -12,6 +12,7 @@ RSpec.describe ActivityPub::ProcessAccountService, type: :service do attachment: [ { type: 'PropertyValue', name: 'Pronouns', value: 'They/them' }, { type: 'PropertyValue', name: 'Occupation', value: 'Unit test' }, + { type: 'PropertyValue', name: 'non-string', value: ['foo', 'bar'] }, ], }.with_indifferent_access end From 85c845c00175a38a6012eeb16542aeba9afb9e75 Mon Sep 17 00:00:00 2001 From: Takeshi Umeda Date: Mon, 9 Aug 2021 20:33:19 +0900 Subject: [PATCH 53/70] Fix invalid blurhash handling in Create activity (#16583) --- app/lib/activitypub/activity/create.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 0c0bfcba0..9607f555e 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -447,10 +447,14 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def supported_blurhash?(blurhash) - components = blurhash.blank? ? nil : Blurhash.components(blurhash) + components = blurhash.blank? || !blurhash_valid_chars?(blurhash) ? nil : Blurhash.components(blurhash) components.present? && components.none? { |comp| comp > 5 } end + def blurhash_valid_chars?(blurhash) + /^[\w#$%*+-.:;=?@\[\]^{|}~]+$/.match?(blurhash) + end + def skip_download? return @skip_download if defined?(@skip_download) From 1ddbefb7876399fcee4ac58a7ff330a0e05d3edf Mon Sep 17 00:00:00 2001 From: Takeshi Umeda Date: Sun, 8 Aug 2021 22:31:02 +0900 Subject: [PATCH 54/70] Fix when MoveWorker cannot get locale from remote account (#16576) --- app/workers/move_worker.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/workers/move_worker.rb b/app/workers/move_worker.rb index 39e321316..a86cc96c5 100644 --- a/app/workers/move_worker.rb +++ b/app/workers/move_worker.rb @@ -41,7 +41,7 @@ class MoveWorker def copy_account_notes! AccountNote.where(target_account: @source_account).find_each do |note| - text = I18n.with_locale(note.account.user.locale || I18n.default_locale) do + text = I18n.with_locale(note.account.user&.locale || I18n.default_locale) do I18n.t('move_handler.copy_account_note_text', acct: @source_account.acct) end @@ -72,7 +72,7 @@ class MoveWorker def add_account_note_if_needed!(account, id) unless AccountNote.where(account: account, target_account: @target_account).exists? - text = I18n.with_locale(account.user.locale || I18n.default_locale) do + text = I18n.with_locale(account.user&.locale || I18n.default_locale) do I18n.t(id, acct: @source_account.acct) end AccountNote.create!(account: account, target_account: @target_account, comment: text) From a19aec0f485a8a04f62a99d569dc21cc5d07a397 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 23 Jun 2021 23:55:47 +0200 Subject: [PATCH 55/70] Fix newlines in accout notes added by the Move handler (#16415) * Fix newlines in account notes added by the move handler * Make MoveWorker more robust --- app/workers/move_worker.rb | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/app/workers/move_worker.rb b/app/workers/move_worker.rb index a86cc96c5..cc2c17d32 100644 --- a/app/workers/move_worker.rb +++ b/app/workers/move_worker.rb @@ -13,9 +13,13 @@ class MoveWorker queue_follow_unfollows! end + @deferred_error = nil + copy_account_notes! carry_blocks_over! carry_mutes_over! + + raise @deferred_error unless @deferred_error.nil? rescue ActiveRecord::RecordNotFound true end @@ -36,6 +40,8 @@ class MoveWorker @source_account.followers.local.select(:id).find_in_batches do |accounts| UnfollowFollowWorker.push_bulk(accounts.map(&:id)) { |follower_id| [follower_id, @source_account.id, @target_account.id, bypass_locked] } + rescue => e + @deferred_error = e end end @@ -47,10 +53,12 @@ class MoveWorker new_note = AccountNote.find_by(account: note.account, target_account: @target_account) if new_note.nil? - AccountNote.create!(account: note.account, target_account: @target_account, comment: [text, note.comment].join('\n')) + AccountNote.create!(account: note.account, target_account: @target_account, comment: [text, note.comment].join("\n")) else - new_note.update!(comment: [text, note.comment, '\n', new_note.comment].join('\n')) + new_note.update!(comment: [text, note.comment, "\n", new_note.comment].join("\n")) end + rescue => e + @deferred_error = e end end @@ -60,6 +68,8 @@ class MoveWorker BlockService.new.call(block.account, @target_account) add_account_note_if_needed!(block.account, 'move_handler.carry_blocks_over_text') end + rescue => e + @deferred_error = e end end @@ -67,6 +77,8 @@ class MoveWorker @source_account.muted_by_relationships.where(account: Account.local).find_each do |mute| MuteService.new.call(mute.account, @target_account, notifications: mute.hide_notifications) unless mute.account.muting?(@target_account) || mute.account.following?(@target_account) add_account_note_if_needed!(mute.account, 'move_handler.carry_mutes_over_text') + rescue => e + @deferred_error = e end end From b6b19419e2d8d8c35ac7b8550fdee693bd80a44a Mon Sep 17 00:00:00 2001 From: Claire Date: Sat, 6 Nov 2021 00:13:58 +0100 Subject: [PATCH 56/70] Fix reviving revoked sessions and invalidating login (#16943) Up until now, we have used Devise's Rememberable mechanism to re-log users after the end of their browser sessions. This mechanism relies on a signed cookie containing a token. That token was stored on the user's record, meaning it was shared across all logged in browsers, meaning truly revoking a browser's ability to auto-log-in involves revoking the token itself, and revoking access from *all* logged-in browsers. We had a session mechanism that dynamically checks whether a user's session has been disabled, and would log out the user if so. However, this would only clear a session being actively used, and a new one could be respawned with the `remember_user_token` cookie. In practice, this caused two issues: - sessions could be revived after being closed from /auth/edit (security issue) - auto-log-in would be disabled for *all* browsers after logging out from one of them This PR removes the `remember_token` mechanism and treats the `_session_id` cookie/token as a browser-specific `remember_token`, fixing both issues. --- app/controllers/auth/passwords_controller.rb | 1 - .../auth/registrations_controller.rb | 3 -- app/controllers/auth/sessions_controller.rb | 2 - app/models/user.rb | 2 +- config/initializers/devise.rb | 39 +++++++++++++++++-- 5 files changed, 37 insertions(+), 10 deletions(-) diff --git a/app/controllers/auth/passwords_controller.rb b/app/controllers/auth/passwords_controller.rb index 5db2668f7..2996c0431 100644 --- a/app/controllers/auth/passwords_controller.rb +++ b/app/controllers/auth/passwords_controller.rb @@ -10,7 +10,6 @@ class Auth::PasswordsController < Devise::PasswordsController super do |resource| if resource.errors.empty? resource.session_activations.destroy_all - resource.forget_me! end end end diff --git a/app/controllers/auth/registrations_controller.rb b/app/controllers/auth/registrations_controller.rb index a3114ab25..3c1730f25 100644 --- a/app/controllers/auth/registrations_controller.rb +++ b/app/controllers/auth/registrations_controller.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true class Auth::RegistrationsController < Devise::RegistrationsController - include Devise::Controllers::Rememberable include RegistrationSpamConcern layout :determine_layout @@ -30,8 +29,6 @@ class Auth::RegistrationsController < Devise::RegistrationsController super do |resource| if resource.saved_change_to_encrypted_password? resource.clear_other_sessions(current_session.session_id) - resource.forget_me! - remember_me(resource) end end end diff --git a/app/controllers/auth/sessions_controller.rb b/app/controllers/auth/sessions_controller.rb index ecb5a0a03..6ebd86037 100644 --- a/app/controllers/auth/sessions_controller.rb +++ b/app/controllers/auth/sessions_controller.rb @@ -1,8 +1,6 @@ # frozen_string_literal: true class Auth::SessionsController < Devise::SessionsController - include Devise::Controllers::Rememberable - layout 'auth' skip_before_action :require_no_authentication, only: [:create] diff --git a/app/models/user.rb b/app/models/user.rb index 5a149f573..a38362e57 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -63,7 +63,7 @@ class User < ApplicationRecord devise :two_factor_backupable, otp_number_of_backup_codes: 10 - devise :registerable, :recoverable, :rememberable, :validatable, + devise :registerable, :recoverable, :validatable, :confirmable include Omniauthable diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb index ef612e177..5232e6cfd 100644 --- a/config/initializers/devise.rb +++ b/config/initializers/devise.rb @@ -1,3 +1,5 @@ +require 'devise/strategies/authenticatable' + Warden::Manager.after_set_user except: :fetch do |user, warden| if user.session_active?(warden.cookies.signed['_session_id'] || warden.raw_session['auth_id']) session_id = warden.cookies.signed['_session_id'] || warden.raw_session['auth_id'] @@ -72,17 +74,48 @@ module Devise mattr_accessor :ldap_uid_conversion_replace @@ldap_uid_conversion_replace = nil - class Strategies::PamAuthenticatable - def valid? - super && ::Devise.pam_authentication + module Strategies + class PamAuthenticatable + def valid? + super && ::Devise.pam_authentication + end + end + + class SessionActivationRememberable < Authenticatable + def valid? + @session_cookie = nil + session_cookie.present? + end + + def authenticate! + resource = SessionActivation.find_by(session_id: session_cookie)&.user + + unless resource + cookies.delete('_session_id') + return pass + end + + if validate(resource) + success!(resource) + end + end + + private + + def session_cookie + @session_cookie ||= cookies.signed['_session_id'] + end end end end +Warden::Strategies.add(:session_activation_rememberable, Devise::Strategies::SessionActivationRememberable) + Devise.setup do |config| config.warden do |manager| manager.default_strategies(scope: :user).unshift :two_factor_ldap_authenticatable if Devise.ldap_authentication manager.default_strategies(scope: :user).unshift :two_factor_pam_authenticatable if Devise.pam_authentication + manager.default_strategies(scope: :user).unshift :session_activation_rememberable manager.default_strategies(scope: :user).unshift :two_factor_authenticatable manager.default_strategies(scope: :user).unshift :two_factor_backupable end From 5d72e6c4d0aca71162c529bf146ff0b59e873198 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 6 Nov 2021 05:17:39 +0100 Subject: [PATCH 57/70] Fix login being broken due to inaccurately applied backport fix in 3.4.2 See #16943 --- app/controllers/auth/sessions_controller.rb | 1 - .../concerns/sign_in_token_authentication_concern.rb | 1 - app/controllers/concerns/two_factor_authentication_concern.rb | 2 -- 3 files changed, 4 deletions(-) diff --git a/app/controllers/auth/sessions_controller.rb b/app/controllers/auth/sessions_controller.rb index 6ebd86037..630b0dee5 100644 --- a/app/controllers/auth/sessions_controller.rb +++ b/app/controllers/auth/sessions_controller.rb @@ -24,7 +24,6 @@ class Auth::SessionsController < Devise::SessionsController def create super do |resource| resource.update_sign_in!(request, new_sign_in: true) - remember_me(resource) flash.delete(:notice) end end diff --git a/app/controllers/concerns/sign_in_token_authentication_concern.rb b/app/controllers/concerns/sign_in_token_authentication_concern.rb index 33ea5cf85..2b5e79115 100644 --- a/app/controllers/concerns/sign_in_token_authentication_concern.rb +++ b/app/controllers/concerns/sign_in_token_authentication_concern.rb @@ -34,7 +34,6 @@ module SignInTokenAuthenticationConcern def authenticate_with_sign_in_token_attempt(user) if valid_sign_in_token_attempt?(user) clear_attempt_from_session - remember_me(user) sign_in(user) else flash.now[:alert] = I18n.t('users.invalid_sign_in_token') diff --git a/app/controllers/concerns/two_factor_authentication_concern.rb b/app/controllers/concerns/two_factor_authentication_concern.rb index 402638eba..d65f072ff 100644 --- a/app/controllers/concerns/two_factor_authentication_concern.rb +++ b/app/controllers/concerns/two_factor_authentication_concern.rb @@ -57,7 +57,6 @@ module TwoFactorAuthenticationConcern if valid_webauthn_credential?(user, webauthn_credential) clear_attempt_from_session - remember_me(user) sign_in(user) render json: { redirect_path: root_path }, status: :ok else @@ -68,7 +67,6 @@ module TwoFactorAuthenticationConcern def authenticate_with_two_factor_via_otp(user) if valid_otp_attempt?(user) clear_attempt_from_session - remember_me(user) sign_in(user) else flash.now[:alert] = I18n.t('users.invalid_otp_token') From e02a7cfeb213a822239a82c87c68be041042421c Mon Sep 17 00:00:00 2001 From: Claire Date: Sat, 6 Nov 2021 00:12:25 +0100 Subject: [PATCH 58/70] Fix AccountNote not having a maximum length (#16942) --- app/models/account_note.rb | 1 + app/workers/move_worker.rb | 8 +++- .../api/v1/accounts/notes_controller_spec.rb | 48 +++++++++++++++++++ spec/workers/move_worker_spec.rb | 41 +++++++++++----- 4 files changed, 86 insertions(+), 12 deletions(-) create mode 100644 spec/controllers/api/v1/accounts/notes_controller_spec.rb diff --git a/app/models/account_note.rb b/app/models/account_note.rb index bf61df923..b338bc92f 100644 --- a/app/models/account_note.rb +++ b/app/models/account_note.rb @@ -17,4 +17,5 @@ class AccountNote < ApplicationRecord belongs_to :target_account, class_name: 'Account' validates :account_id, uniqueness: { scope: :target_account_id } + validates :comment, length: { maximum: 2_000 } end diff --git a/app/workers/move_worker.rb b/app/workers/move_worker.rb index cc2c17d32..4a900e3b8 100644 --- a/app/workers/move_worker.rb +++ b/app/workers/move_worker.rb @@ -53,10 +53,16 @@ class MoveWorker new_note = AccountNote.find_by(account: note.account, target_account: @target_account) if new_note.nil? - AccountNote.create!(account: note.account, target_account: @target_account, comment: [text, note.comment].join("\n")) + begin + AccountNote.create!(account: note.account, target_account: @target_account, comment: [text, note.comment].join("\n")) + rescue ActiveRecord::RecordInvalid + AccountNote.create!(account: note.account, target_account: @target_account, comment: note.comment) + end else new_note.update!(comment: [text, note.comment, "\n", new_note.comment].join("\n")) end + rescue ActiveRecord::RecordInvalid + nil rescue => e @deferred_error = e end diff --git a/spec/controllers/api/v1/accounts/notes_controller_spec.rb b/spec/controllers/api/v1/accounts/notes_controller_spec.rb new file mode 100644 index 000000000..0a2957fed --- /dev/null +++ b/spec/controllers/api/v1/accounts/notes_controller_spec.rb @@ -0,0 +1,48 @@ +require 'rails_helper' + +describe Api::V1::Accounts::NotesController do + render_views + + let(:user) { Fabricate(:user, account: Fabricate(:account, username: 'alice')) } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'write:accounts') } + let(:account) { Fabricate(:account) } + let(:comment) { 'foo' } + + before do + allow(controller).to receive(:doorkeeper_token) { token } + end + + describe 'POST #create' do + subject do + post :create, params: { account_id: account.id, comment: comment } + end + + context 'when account note has reasonable length' do + let(:comment) { 'foo' } + + it 'returns http success' do + subject + expect(response).to have_http_status(200) + end + + it 'updates account note' do + subject + expect(AccountNote.find_by(account_id: user.account.id, target_account_id: account.id).comment).to eq comment + end + end + + context 'when account note exceends allowed length' do + let(:comment) { 'a' * 2_001 } + + it 'returns 422' do + subject + expect(response).to have_http_status(422) + end + + it 'does not create account note' do + subject + expect(AccountNote.where(account_id: user.account.id, target_account_id: account.id).exists?).to be_falsey + end + end + end +end diff --git a/spec/workers/move_worker_spec.rb b/spec/workers/move_worker_spec.rb index 8ab4f182f..82449b0c7 100644 --- a/spec/workers/move_worker_spec.rb +++ b/spec/workers/move_worker_spec.rb @@ -9,7 +9,8 @@ describe MoveWorker do let(:source_account) { Fabricate(:account, protocol: :activitypub, domain: 'example.com') } let(:target_account) { Fabricate(:account, protocol: :activitypub, domain: 'example.com') } let(:local_user) { Fabricate(:user) } - let!(:account_note) { Fabricate(:account_note, account: local_user.account, target_account: source_account) } + let(:comment) { 'old note prior to move' } + let!(:account_note) { Fabricate(:account_note, account: local_user.account, target_account: source_account, comment: comment) } let(:block_service) { double } @@ -26,19 +27,37 @@ describe MoveWorker do end shared_examples 'user note handling' do - it 'copies user note' do - subject.perform(source_account.id, target_account.id) - expect(AccountNote.find_by(account: account_note.account, target_account: target_account).comment).to include(source_account.acct) - expect(AccountNote.find_by(account: account_note.account, target_account: target_account).comment).to include(account_note.comment) + context 'when user notes are short enough' do + it 'copies user note with prelude' do + subject.perform(source_account.id, target_account.id) + expect(AccountNote.find_by(account: account_note.account, target_account: target_account).comment).to include(source_account.acct) + expect(AccountNote.find_by(account: account_note.account, target_account: target_account).comment).to include(account_note.comment) + end + + it 'merges user notes when needed' do + new_account_note = AccountNote.create!(account: account_note.account, target_account: target_account, comment: 'new note prior to move') + + subject.perform(source_account.id, target_account.id) + expect(AccountNote.find_by(account: account_note.account, target_account: target_account).comment).to include(source_account.acct) + expect(AccountNote.find_by(account: account_note.account, target_account: target_account).comment).to include(account_note.comment) + expect(AccountNote.find_by(account: account_note.account, target_account: target_account).comment).to include(new_account_note.comment) + end end - it 'merges user notes when needed' do - new_account_note = AccountNote.create!(account: account_note.account, target_account: target_account, comment: 'new note prior to move') + context 'when user notes are too long' do + let(:comment) { 'abc' * 333 } - subject.perform(source_account.id, target_account.id) - expect(AccountNote.find_by(account: account_note.account, target_account: target_account).comment).to include(source_account.acct) - expect(AccountNote.find_by(account: account_note.account, target_account: target_account).comment).to include(account_note.comment) - expect(AccountNote.find_by(account: account_note.account, target_account: target_account).comment).to include(new_account_note.comment) + it 'copies user note without prelude' do + subject.perform(source_account.id, target_account.id) + expect(AccountNote.find_by(account: account_note.account, target_account: target_account).comment).to include(account_note.comment) + end + + it 'keeps user notes unchanged' do + new_account_note = AccountNote.create!(account: account_note.account, target_account: target_account, comment: 'new note prior to move') + + subject.perform(source_account.id, target_account.id) + expect(AccountNote.find_by(account: account_note.account, target_account: target_account).comment).to include(new_account_note.comment) + end end end From ff19501e50affda3e5ec81fa1c0c16bea178a86b Mon Sep 17 00:00:00 2001 From: Claire Date: Sun, 14 Nov 2021 21:55:40 +0100 Subject: [PATCH 59/70] Fix confusing error when webfinger request returns empty document (#16986) For some reason, some misconfigured servers return an empty document when queried over webfinger. Since an empty document does not lead to a parse error, the error is not caught properly and triggers uncaught exceptions later on. This PR fixes that by immediately erroring out with `Webfinger::Error` on getting an empty response. --- app/lib/webfinger.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/lib/webfinger.rb b/app/lib/webfinger.rb index c7aa43bb3..06fac8c37 100644 --- a/app/lib/webfinger.rb +++ b/app/lib/webfinger.rb @@ -46,7 +46,9 @@ class Webfinger def body_from_webfinger(url = standard_url, use_fallback = true) webfinger_request(url).perform do |res| if res.code == 200 - res.body_with_limit + body = res.body_with_limit + raise Webfinger::Error, "Request for #{@uri} returned empty response" if body.empty? + body elsif res.code == 404 && use_fallback body_from_host_meta elsif res.code == 410 From dbe5e29e3896dacef2c4f0142e1e2b22a6d43d8b Mon Sep 17 00:00:00 2001 From: Claire Date: Tue, 16 Nov 2021 21:36:28 +0100 Subject: [PATCH 60/70] Fix upload of remote media with OpenStack Swift sometimes failing (#16998) Under certain conditions, files fetched from remotes trigger an error when being uploaded using OpenStack Swift. This is because in some cases, the remote server will not return a content-length, so our ResponseWithLimitAdapter will hold a `nil` value for `#size`, which will lead to an invalid value for the Content-Length header of the Swift API call. This commit fixes that by taking the size from the actually-downloaded file size rather than the upstream-provided Content-Length header value. --- lib/paperclip/response_with_limit_adapter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/paperclip/response_with_limit_adapter.rb b/lib/paperclip/response_with_limit_adapter.rb index 17a2abd25..deb89717a 100644 --- a/lib/paperclip/response_with_limit_adapter.rb +++ b/lib/paperclip/response_with_limit_adapter.rb @@ -17,9 +17,9 @@ module Paperclip def cache_current_values @original_filename = filename_from_content_disposition.presence || filename_from_path.presence || 'data' - @size = @target.response.content_length @tempfile = copy_to_tempfile(@target) @content_type = ContentTypeDetector.new(@tempfile.path).detect + @size = File.size(@tempfile) end def copy_to_tempfile(source) From 82a490ac7f92c1bae40534c878cb4ecf7776b8b4 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 25 Nov 2021 23:46:30 +0100 Subject: [PATCH 61/70] Fix filtering DMs from non-followed users (#17042) --- app/services/notify_service.rb | 43 +++++++++++++++++++++++++++- spec/services/notify_service_spec.rb | 17 +++++++++-- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/app/services/notify_service.rb b/app/services/notify_service.rb index fc187db40..e78c74d1e 100644 --- a/app/services/notify_service.rb +++ b/app/services/notify_service.rb @@ -67,8 +67,49 @@ class NotifyService < BaseService message? && @notification.target_status.direct_visibility? end + # Returns true if the sender has been mentionned by the recipient up the thread def response_to_recipient? - @notification.target_status.in_reply_to_account_id == @recipient.id && @notification.target_status.thread&.direct_visibility? + return false if @notification.target_status.in_reply_to_id.nil? + + # Using an SQL CTE to avoid unneeded back-and-forth with SQL server in case of long threads + !Status.count_by_sql([<<-SQL.squish, id: @notification.target_status.in_reply_to_id, recipient_id: @recipient.id, sender_id: @notification.from_account.id]).zero? + WITH RECURSIVE ancestors(id, in_reply_to_id, replying_to_sender) AS ( + SELECT + s.id, s.in_reply_to_id, (CASE + WHEN s.account_id = :recipient_id THEN + EXISTS ( + SELECT * + FROM mentions m + WHERE m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id + ) + ELSE + FALSE + END) + FROM statuses s + WHERE s.id = :id + UNION ALL + SELECT + s.id, + s.in_reply_to_id, + (CASE + WHEN s.account_id = :recipient_id THEN + EXISTS ( + SELECT * + FROM mentions m + WHERE m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id + ) + ELSE + FALSE + END) + FROM ancestors st + JOIN statuses s ON s.id = st.in_reply_to_id + WHERE st.replying_to_sender IS FALSE + ) + SELECT COUNT(*) + FROM ancestors st + JOIN statuses s ON s.id = st.id + WHERE st.replying_to_sender IS TRUE AND s.visibility = 3 + SQL end def from_staff? diff --git a/spec/services/notify_service_spec.rb b/spec/services/notify_service_spec.rb index f2cb22c5e..7433866b7 100644 --- a/spec/services/notify_service_spec.rb +++ b/spec/services/notify_service_spec.rb @@ -64,8 +64,9 @@ RSpec.describe NotifyService, type: :service do is_expected.to_not change(Notification, :count) end - context 'if the message chain initiated by recipient, but is not direct message' do + context 'if the message chain is initiated by recipient, but is not direct message' do let(:reply_to) { Fabricate(:status, account: recipient) } + let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) } let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: reply_to)) } it 'does not notify' do @@ -73,8 +74,20 @@ RSpec.describe NotifyService, type: :service do end end - context 'if the message chain initiated by recipient and is direct message' do + context 'if the message chain is initiated by recipient, but without a mention to the sender, even if the sender sends multiple messages in a row' do + let(:reply_to) { Fabricate(:status, account: recipient) } + let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) } + let(:dummy_reply) { Fabricate(:status, account: sender, visibility: :direct, thread: reply_to) } + let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: dummy_reply)) } + + it 'does not notify' do + is_expected.to_not change(Notification, :count) + end + end + + context 'if the message chain is initiated by the recipient with a mention to the sender' do let(:reply_to) { Fabricate(:status, account: recipient, visibility: :direct) } + let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) } let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: reply_to)) } it 'does notify' do From ad06423e71106ee2fb63162b2ab6d8acf6861416 Mon Sep 17 00:00:00 2001 From: Claire Date: Sun, 30 Jan 2022 21:38:54 +0100 Subject: [PATCH 62/70] Fix some old migration scripts (#17394) * Fix some old migration scripts * Fix edge case in two-step migration from older releases --- ...3_remove_faux_remote_account_duplicates.rb | 40 +++++++++++++++++++ .../20190715164535_add_instance_actor.rb | 5 +++ .../20191007013357_update_pt_locales.rb | 4 ++ 3 files changed, 49 insertions(+) diff --git a/db/migrate/20181026034033_remove_faux_remote_account_duplicates.rb b/db/migrate/20181026034033_remove_faux_remote_account_duplicates.rb index bd4f4c2a3..40537e9c9 100644 --- a/db/migrate/20181026034033_remove_faux_remote_account_duplicates.rb +++ b/db/migrate/20181026034033_remove_faux_remote_account_duplicates.rb @@ -1,6 +1,46 @@ class RemoveFauxRemoteAccountDuplicates < ActiveRecord::Migration[5.2] disable_ddl_transaction! + class StreamEntry < ApplicationRecord + # Dummy class, to make migration possible across version changes + belongs_to :account, inverse_of: :stream_entries + end + + class Status < ApplicationRecord + # Dummy class, to make migration possible across version changes + belongs_to :account, inverse_of: :statuses + has_many :favourites, inverse_of: :status, dependent: :destroy + has_many :mentions, dependent: :destroy, inverse_of: :status + end + + class Favourite < ApplicationRecord + # Dummy class, to make migration possible across version changes + belongs_to :account, inverse_of: :favourites + belongs_to :status, inverse_of: :favourites + end + + class Mention < ApplicationRecord + # Dummy class, to make migration possible across version changes + belongs_to :account, inverse_of: :mentions + belongs_to :status + end + + class Notification < ApplicationRecord + # Dummy class, to make migration possible across version changes + belongs_to :account, optional: true + belongs_to :from_account, class_name: 'Account', optional: true + belongs_to :activity, polymorphic: true, optional: true + end + + class Account < ApplicationRecord + # Dummy class, to make migration possible across version changes + has_many :stream_entries, inverse_of: :account, dependent: :destroy + has_many :statuses, inverse_of: :account, dependent: :destroy + has_many :favourites, inverse_of: :account, dependent: :destroy + has_many :mentions, inverse_of: :account, dependent: :destroy + has_many :notifications, inverse_of: :account, dependent: :destroy + end + def up local_domain = Rails.configuration.x.local_domain diff --git a/db/migrate/20190715164535_add_instance_actor.rb b/db/migrate/20190715164535_add_instance_actor.rb index a26d54949..8c0301d69 100644 --- a/db/migrate/20190715164535_add_instance_actor.rb +++ b/db/migrate/20190715164535_add_instance_actor.rb @@ -1,4 +1,9 @@ class AddInstanceActor < ActiveRecord::Migration[5.2] + class Account < ApplicationRecord + # Dummy class, to make migration possible across version changes + validates :username, uniqueness: { scope: :domain, case_sensitive: false } + end + def up Account.create!(id: -99, actor_type: 'Application', locked: true, username: Rails.configuration.x.local_domain) end diff --git a/db/migrate/20191007013357_update_pt_locales.rb b/db/migrate/20191007013357_update_pt_locales.rb index b7288d38a..9e8f8b424 100644 --- a/db/migrate/20191007013357_update_pt_locales.rb +++ b/db/migrate/20191007013357_update_pt_locales.rb @@ -1,4 +1,8 @@ class UpdatePtLocales < ActiveRecord::Migration[5.2] + class User < ApplicationRecord + # Dummy class, to make migration possible across version changes + end + disable_ddl_transaction! def up From 5ec943f85d879ee26bfd8c4efac3cbd70ed9488c Mon Sep 17 00:00:00 2001 From: Claire Date: Sun, 30 Jan 2022 22:34:54 +0100 Subject: [PATCH 63/70] Fix edge case in migration helpers that caused crash because of PostgreSQL quirks (#17398) --- lib/mastodon/migration_helpers.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mastodon/migration_helpers.rb b/lib/mastodon/migration_helpers.rb index 30645b923..ab2318a72 100644 --- a/lib/mastodon/migration_helpers.rb +++ b/lib/mastodon/migration_helpers.rb @@ -329,7 +329,7 @@ module Mastodon table = Arel::Table.new(table_name) total = estimate_rows_in_table(table_name).to_i - if total == 0 + if total < 1 count_arel = table.project(Arel.star.count.as('count')) count_arel = yield table, count_arel if block_given? From 7619689fcbc8ff30e22bb57131c2683ff0303c09 Mon Sep 17 00:00:00 2001 From: Claire Date: Sun, 30 Jan 2022 17:12:02 +0100 Subject: [PATCH 64/70] Add more advanced migration tests - populate the database with some data when testing migrations - try both one-step and two-step migrations (`SKIP_POST_DEPLOYMENT_MIGRATIONS`) --- .circleci/config.yml | 42 +++++++++- lib/tasks/tests.rake | 181 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 222 insertions(+), 1 deletion(-) create mode 100644 lib/tasks/tests.rake diff --git a/.circleci/config.yml b/.circleci/config.yml index 862fa126b..42d9b0b48 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -160,8 +160,45 @@ jobs: name: Create database command: ./bin/rails db:create - run: - name: Run migrations + command: ./bin/rails db:migrate VERSION=20171010025614 + name: Run migrations up to v2.0.0 + - run: + command: ./bin/rails tests:migrations:populate_v2 + name: Populate database with test data + - run: command: ./bin/rails db:migrate + name: Run all remaining migrations + + test-two-step-migrations: + <<: *defaults + docker: + - image: circleci/ruby:2.7-buster-node + environment: *ruby_environment + - image: circleci/postgres:12.2 + environment: + POSTGRES_USER: root + POSTGRES_HOST_AUTH_METHOD: trust + - image: circleci/redis:5-alpine + steps: + - *attach_workspace + - *install_system_dependencies + - run: + command: ./bin/rails db:create + name: Create database + - run: + command: ./bin/rails db:migrate VERSION=20171010025614 + name: Run migrations up to v2.0.0 + - run: + command: ./bin/rails tests:migrations:populate_v2 + name: Populate database with test data + - run: + command: ./bin/rails db:migrate + name: Run all pre-deployment migrations + evironment: + SKIP_POST_DEPLOYMENT_MIGRATIONS: true + - run: + command: ./bin/rails db:migrate + name: Run all post-deployment remaining migrations test-ruby2.7: <<: *defaults @@ -233,6 +270,9 @@ workflows: - test-migrations: requires: - install-ruby2.7 + - test-two-step-migrations: + requires: + - install-ruby2.7 - test-ruby2.7: requires: - install-ruby2.7 diff --git a/lib/tasks/tests.rake b/lib/tasks/tests.rake new file mode 100644 index 000000000..0f38b50e3 --- /dev/null +++ b/lib/tasks/tests.rake @@ -0,0 +1,181 @@ +# frozen_string_literal: true + +namespace :tests do + namespace :migrations do + desc 'Populate the database with test data for 2.0.0' + task populate_v2: :environment do + admin_key = OpenSSL::PKey::RSA.new(2048) + user_key = OpenSSL::PKey::RSA.new(2048) + remote_key = OpenSSL::PKey::RSA.new(2048) + remote_key2 = OpenSSL::PKey::RSA.new(2048) + remote_key3 = OpenSSL::PKey::RSA.new(2048) + admin_private_key = ActiveRecord::Base.connection.quote(admin_key.to_pem) + admin_public_key = ActiveRecord::Base.connection.quote(admin_key.public_key.to_pem) + user_private_key = ActiveRecord::Base.connection.quote(user_key.to_pem) + user_public_key = ActiveRecord::Base.connection.quote(user_key.public_key.to_pem) + remote_public_key = ActiveRecord::Base.connection.quote(remote_key.public_key.to_pem) + remote_public_key2 = ActiveRecord::Base.connection.quote(remote_key2.public_key.to_pem) + remote_public_key_ap = ActiveRecord::Base.connection.quote(remote_key3.public_key.to_pem) + local_domain = ActiveRecord::Base.connection.quote(Rails.configuration.x.local_domain) + + ActiveRecord::Base.connection.execute(<<~SQL) + -- accounts + + INSERT INTO "accounts" + (id, username, domain, private_key, public_key, created_at, updated_at) + VALUES + (1, 'admin', NULL, #{admin_private_key}, #{admin_public_key}, now(), now()), + (2, 'user', NULL, #{user_private_key}, #{user_public_key}, now(), now()); + + INSERT INTO "accounts" + (id, username, domain, private_key, public_key, created_at, updated_at, remote_url, salmon_url) + VALUES + (3, 'remote', 'remote.com', NULL, #{remote_public_key}, now(), now(), + 'https://remote.com/@remote', 'https://remote.com/salmon/1'), + (4, 'Remote', 'remote.com', NULL, #{remote_public_key}, now(), now(), + 'https://remote.com/@Remote', 'https://remote.com/salmon/1'), + (5, 'REMOTE', 'Remote.com', NULL, #{remote_public_key2}, now(), now(), + 'https://remote.com/stale/@REMOTE', 'https://remote.com/stale/salmon/1'); + + INSERT INTO "accounts" + (id, username, domain, private_key, public_key, created_at, updated_at, protocol, inbox_url, outbox_url, followers_url) + VALUES + (6, 'bob', 'activitypub.com', NULL, #{remote_public_key_ap}, now(), now(), + 1, 'https://activitypub.com/users/bob/inbox', 'https://activitypub.com/users/bob/outbox', 'https://activitypub.com/users/bob/followers'); + + INSERT INTO "accounts" + (id, username, domain, private_key, public_key, created_at, updated_at) + VALUES + (7, 'user', #{local_domain}, #{user_private_key}, #{user_public_key}, now(), now()), + (8, 'pt_user', NULL, #{user_private_key}, #{user_public_key}, now(), now()); + + -- users + + INSERT INTO "users" + (id, account_id, email, created_at, updated_at, admin) + VALUES + (1, 1, 'admin@localhost', now(), now(), true), + (2, 2, 'user@localhost', now(), now(), false); + + INSERT INTO "users" + (id, account_id, email, created_at, updated_at, admin, locale) + VALUES + (3, 7, 'ptuser@localhost', now(), now(), false, 'pt'); + + -- statuses + + INSERT INTO "statuses" + (id, account_id, text, created_at, updated_at) + VALUES + (1, 1, 'test', now(), now()), + (2, 1, '@remote@remote.com hello', now(), now()), + (3, 1, '@Remote@remote.com hello', now(), now()), + (4, 1, '@REMOTE@remote.com hello', now(), now()); + + INSERT INTO "statuses" + (id, account_id, text, created_at, updated_at, uri, local) + VALUES + (5, 1, 'activitypub status', now(), now(), 'https://localhost/users/admin/statuses/4', true); + + INSERT INTO "statuses" + (id, account_id, text, created_at, updated_at) + VALUES + (6, 3, 'test', now(), now()); + + INSERT INTO "statuses" + (id, account_id, text, created_at, updated_at, in_reply_to_id, in_reply_to_account_id) + VALUES + (7, 4, '@admin hello', now(), now(), 3, 1); + + INSERT INTO "statuses" + (id, account_id, text, created_at, updated_at) + VALUES + (8, 5, 'test', now(), now()); + + INSERT INTO "statuses" + (id, account_id, reblog_of_id, created_at, updated_at) + VALUES + (9, 1, 2, now(), now()); + + -- mentions (from previous statuses) + + INSERT INTO "mentions" + (status_id, account_id, created_at, updated_at) + VALUES + (2, 3, now(), now()), + (3, 4, now(), now()), + (4, 5, now(), now()); + + -- stream entries + + INSERT INTO "stream_entries" + (activity_id, account_id, activity_type, created_at, updated_at) + VALUES + (1, 1, 'status', now(), now()), + (2, 1, 'status', now(), now()), + (3, 1, 'status', now(), now()), + (4, 1, 'status', now(), now()), + (5, 1, 'status', now(), now()), + (6, 3, 'status', now(), now()), + (7, 4, 'status', now(), now()), + (8, 5, 'status', now(), now()), + (9, 1, 'status', now(), now()); + + + -- custom emoji + + INSERT INTO "custom_emojis" + (shortcode, created_at, updated_at) + VALUES + ('test', now(), now()), + ('Test', now(), now()), + ('blobcat', now(), now()); + + INSERT INTO "custom_emojis" + (shortcode, domain, uri, created_at, updated_at) + VALUES + ('blobcat', 'remote.org', 'https://remote.org/emoji/blobcat', now(), now()), + ('blobcat', 'Remote.org', 'https://remote.org/emoji/blobcat', now(), now()), + ('Blobcat', 'remote.org', 'https://remote.org/emoji/Blobcat', now(), now()); + + -- favourites + + INSERT INTO "favourites" + (account_id, status_id, created_at, updated_at) + VALUES + (1, 1, now(), now()), + (1, 7, now(), now()), + (4, 1, now(), now()), + (3, 1, now(), now()), + (5, 1, now(), now()); + + -- pinned statuses + + INSERT INTO "status_pins" + (account_id, status_id, created_at, updated_at) + VALUES + (1, 1, now(), now()), + (3, 6, now(), now()), + (4, 7, now(), now()); + + -- follows + + INSERT INTO "follows" + (account_id, target_account_id, created_at, updated_at) + VALUES + (1, 5, now(), now()), + (6, 2, now(), now()), + (5, 2, now(), now()), + (6, 1, now(), now()); + + -- follow requests + + INSERT INTO "follow_requests" + (account_id, target_account_id, created_at, updated_at) + VALUES + (2, 5, now(), now()), + (5, 1, now(), now()); + SQL + end + end +end From c2bd6e90b43d51a930828526175a51c7f87e1c56 Mon Sep 17 00:00:00 2001 From: Yusuke Nakamura Date: Fri, 12 Nov 2021 13:18:29 +0900 Subject: [PATCH 65/70] Build container image by GitHub Actions (#16973) * Build container image by GitHub Actions * Trigger docker build only pushed to main branch * Tweak tagging imgae - "edge" is the main branch - "latest" is the tagged latest release --- .github/workflows/build-image.yml | 34 +++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/build-image.yml diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml new file mode 100644 index 000000000..2ec6b3d4b --- /dev/null +++ b/.github/workflows/build-image.yml @@ -0,0 +1,34 @@ +name: Build container image +on: + push: + branches: + - "main" + tags: + - "*" +jobs: + build-image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: docker/setup-buildx-action@v1 + - uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: docker/metadata-action@v3 + id: meta + with: + images: ghcr.io/${{ github.repository_owner }}/mastodon + flavor: | + latest=true + tags: | + type=edge,branch=main + type=semver,pattern={{ raw }} + - uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/mastodon:latest + cache-to: type=inline From df6a953f520b2346ecf4ddfcb394bd0260e8182c Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 14 Nov 2021 06:11:05 +0100 Subject: [PATCH 66/70] Change workflow to push to Docker Hub (#16980) --- .github/workflows/build-image.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 2ec6b3d4b..ae4c2aff9 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -13,15 +13,14 @@ jobs: - uses: docker/setup-buildx-action@v1 - uses: docker/login-action@v1 with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - uses: docker/metadata-action@v3 id: meta with: - images: ghcr.io/${{ github.repository_owner }}/mastodon + images: tootsuite/mastodon flavor: | - latest=true + latest=auto tags: | type=edge,branch=main type=semver,pattern={{ raw }} @@ -30,5 +29,5 @@ jobs: context: . push: true tags: ${{ steps.meta.outputs.tags }} - cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/mastodon:latest + cache-from: type=registry,ref=tootsuite/mastodon:latest cache-to: type=inline From e8a2d12338b09ee3a597c9b1458d35859f93c50a Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 16 Nov 2021 21:42:14 +0100 Subject: [PATCH 67/70] Add manual GitHub Actions runs (#17000) --- .github/workflows/build-image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index ae4c2aff9..58f2813d3 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -1,5 +1,6 @@ name: Build container image on: + workflow_dispatch: push: branches: - "main" From 0842e3b4fb368edc3db14f4e82e7686863e3e824 Mon Sep 17 00:00:00 2001 From: Jeong Arm Date: Sun, 26 Dec 2021 06:52:24 +0900 Subject: [PATCH 68/70] Save bundle config as local (#17188) Some bundle options are saved as global user config and not project local. Specially, `deployment` must be saved as local config to be run on copied environment --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 95d45bab4..d472c4b39 100644 --- a/Dockerfile +++ b/Dockerfile @@ -71,8 +71,8 @@ RUN npm install -g yarn && \ COPY Gemfile* package.json yarn.lock /opt/mastodon/ RUN cd /opt/mastodon && \ - bundle config set deployment 'true' && \ - bundle config set without 'development test' && \ + bundle config set --local deployment 'true' && \ + bundle config set --local without 'development test' && \ bundle install -j$(nproc) && \ yarn install --pure-lockfile From b84182b5ba998890db89eb839195caaae615dda6 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 28 Jan 2022 22:00:41 +0100 Subject: [PATCH 69/70] Bump to version 3.3.1 --- CHANGELOG.md | 66 +++++++++++++++++++++++++++++++++++++++++ lib/mastodon/version.rb | 2 +- 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d749c255..658447ba8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,72 @@ Changelog All notable changes to this project will be documented in this file. +## [3.3.1] - 2022-01-31 +### Added +- Add more advanced migration tests ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17393)) +- Add github workflow to build Docker images ([unasuke](https://github.com/mastodon/mastodon/pull/16973), [Gargron](https://github.com/mastodon/mastodon/pull/16980), [Gargron](https://github.com/mastodon/mastodon/pull/17000)) + +### Fixed +- Update some dependencies that were broken or unavailable +- Fix some old migrations failing when skipping releases ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17394)) +- Fix migrations script failing in certain edge cases ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17398)) +- Fix media redownload worker retrying on unexpected response codes ([Gargron](https://github.com/tootsuite/mastodon/pull/16111)) +- Fix thread resolve worker retrying when status no longer exists ([Gargron](https://github.com/tootsuite/mastodon/pull/16109)) +- Fix n+1 queries when rendering statuses in REST API ([abcang](https://github.com/tootsuite/mastodon/pull/15641)) +- Fix remote reporters not receiving suspend/unsuspend activities ([Gargron](https://github.com/tootsuite/mastodon/pull/16050)) +- Fix understanding (not fully qualified) `as:Public` and `Public` ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15948)) +- Fix actor update not being distributed on profile picture deletion ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15461)) +- Fix processing of incoming Delete activities ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/16084)) +- Fix processing of incoming Block activities ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15546)) +- Fix processing of incoming Update activities of unknown accounts ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15514)) +- Fix URIs of repeat follow requests not being recorded ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15662)) +- Fix error on requests with no `Digest` header ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15782)) +- Fix activity object not requiring signature in secure mode ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15592)) +- Fix database serialization failure returning HTTP 500 ([Gargron](https://github.com/tootsuite/mastodon/pull/16101)) +- Fix media processing getting stuck on too much stdin/stderr ([Gargron](https://github.com/tootsuite/mastodon/pull/16136)) +- Fix `tootctl maintenance fix-duplicates` failures ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15923), [ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15515)) +- Fix blocking someone not clearing up list feeds ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/16205)) +- Fix edge case where follow limit interferes with accepting a follow ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/16098)) +- Fix reports of already suspended accounts being recorded ([Gargron](https://github.com/tootsuite/mastodon/pull/16047)) +- Fix sign-up restrictions based on IP addresses not being enforced ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15607)) +- Fix race conditions on account migration creation ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15597)) +- Fix not being able to change world filter expiration back to “Never” ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15858)) +- Fix error when muting users with `duration` in REST API ([Tak](https://github.com/tootsuite/mastodon/pull/15516)) +- Fix wrong URL to custom CSS when `CDN_HOST` is used ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15927)) +- Fix `tootctl accounts unfollow` ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15639)) +- Fix `tootctl emoji import` wasting time on MacOS shadow files ([cortices](https://github.com/tootsuite/mastodon/pull/15430)) +- Fix `tootctl emoji import` not treating shortcodes as case-insensitive ([angristan](https://github.com/tootsuite/mastodon/pull/15738)) +- Fix some issues with SAML account creation ([Gargron](https://github.com/tootsuite/mastodon/pull/15222), [kaiyou](https://github.com/tootsuite/mastodon/pull/15511 +- Fix instance actor not being automatically created if it wasn't seeded properly ([ClearlyClaire](https://github.com/tootsuite/mastodon/pull/15693)))) +- Fix app name, website and redirect URIs not having a maximum length ([Gargron](https://github.com/tootsuite/mastodon/pull/16042)) +- Fix some ActivityPub identifiers in server actor outbox ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16343)) +- Fix custom CSS path setting cookies and being uncacheable due to it ([tribela](https://github.com/mastodon/mastodon/pull/16314)) +- Fix some redis locks auto-releasing too fast ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16276), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/16291)) +- Fix migration script not being able to run if it fails midway ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16312)) +- Fix PWA not being usable from alternate domains ([HolgerHuo](https://github.com/mastodon/mastodon/pull/16714)) +- Fix scheduling a status decreasing status count ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16791)) +- Fix followers synchronization mechanism not working when URI has empty path ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16744)) +- Fix serialization of counts in REST API when user hides their network ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16418)) +- Fix `tootctl self-destruct` not sending delete activities for recently-suspended accounts ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16688)) +- Fix `mastodon:setup` generated env-file syntax ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16896), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/16976)) +- Fix link previews being incorrectly generated from earlier links ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16885)) +- Fix wrong `to`/`cc` values for remote groups in ActivityPub ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16700)) +- Fix mentions with non-ascii TLDs not being processed ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16689)) +- Fix authentication failures halfway through a sign-in attempt ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16607), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/16792)) +- Fix suspended accounts statuses being merged back into timelines ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16628)) +- Fix crash when encountering invalid account fields ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16598)) +- Fix invalid blurhash handling for remote activities ([noellabo](https://github.com/mastodon/mastodon/pull/16583)) +- Fix newlines being added to accout notes when an account moves ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16415), [noellabo](https://github.com/mastodon/mastodon/pull/16576)) +- Fix logging out from one browser logging out all other sessions ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16943)) +- Fix confusing error when WebFinger request returns empty document ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16986)) +- Fix upload of remote media with OpenStack Swift sometimes failing ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16998)) +- Fix Docker build ([tribela](https://github.com/mastodon/mastodon/pull/17188)) + +### Security +- Fix user notes not having a length limit ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16942)) +- Fix revoking a specific session not working ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16943)) +- Fix filtering DMs from non-followed users ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17042)) + ## [3.3.0] - 2020-12-27 ### Added diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index bd0915775..1e298b807 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 0 + 1 end def flags From b8a5b3a3db2ef4d67f5fe4cc553c16b5407f8cb3 Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 31 Jan 2022 18:16:17 +0100 Subject: [PATCH 70/70] Change docker-compose.yml to specifically tag v3.3.1 images --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 52eea7a74..a92b40c22 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,7 +43,7 @@ services: web: build: . - image: tootsuite/mastodon + image: tootsuite/mastodon:v3.3.1 restart: always env_file: .env.production command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000" @@ -63,7 +63,7 @@ services: streaming: build: . - image: tootsuite/mastodon + image: tootsuite/mastodon:v3.3.1 restart: always env_file: .env.production command: node ./streaming @@ -80,7 +80,7 @@ services: sidekiq: build: . - image: tootsuite/mastodon + image: tootsuite/mastodon:v3.3.1 restart: always env_file: .env.production command: bundle exec sidekiq