From aecdaf5a8c001a6e0e75a20072564de754ab5f8b Mon Sep 17 00:00:00 2001 From: ThibG Date: Mon, 14 Sep 2020 13:04:29 +0200 Subject: [PATCH 01/23] Do not serve account actors at all in limited federation mode (#14800) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Do not serve account actors at all in limited federation mode When an account is fetched without a signature from an allowed instance, return an error. This isn't really an improvement in security, as the only information that was previously returned was required protocol-level info, and the only personal bit was the existence of the account. The existence of the account can still be checked by issuing a webfinger query, as those are accepted without signatures. However, this change makes it so that unallowed instances won't create account records on their end when they find a reference to an unknown account. The previous behavior of rendering a limited list of fields, instead of not rendering the actor at all, was in order to prevent situations in which two instances in Authorized Fetch mode or Limited Federation mode would fail to reach each other because resolving an account would require a signed query… from an account which can only be fetched with a signed query itself. However, this should now be fine as fetching accounts is done by signing on behalf of the special instance actor, which does not require any kind of valid signature to be fetched. * Fix tests --- app/controllers/accounts_controller.rb | 11 ++--------- spec/controllers/accounts_controller_spec.rb | 20 ++------------------ 2 files changed, 4 insertions(+), 27 deletions(-) diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index db77b628c..f5e82692e 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -7,6 +7,7 @@ class AccountsController < ApplicationController include AccountControllerConcern include SignatureAuthentication + before_action :require_signature!, if: -> { request.format == :json && authorized_fetch_mode? } before_action :set_cache_headers before_action :set_body_classes @@ -49,7 +50,7 @@ class AccountsController < ApplicationController format.json do expires_in 3.minutes, public: !(authorized_fetch_mode? && signed_request_account.present?) - render_with_cache json: @account, content_type: 'application/activity+json', serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter, fields: restrict_fields_to + render_with_cache json: @account, content_type: 'application/activity+json', serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter end end end @@ -149,12 +150,4 @@ class AccountsController < ApplicationController def params_slice(*keys) params.slice(*keys).permit(*keys) end - - def restrict_fields_to - if signed_request_account.present? || public_fetch_mode? - # Return all fields - else - %i(id type preferred_username inbox public_key endpoints) - end - end end diff --git a/spec/controllers/accounts_controller_spec.rb b/spec/controllers/accounts_controller_spec.rb index 93bf2c83f..b04f4650b 100644 --- a/spec/controllers/accounts_controller_spec.rb +++ b/spec/controllers/accounts_controller_spec.rb @@ -348,24 +348,8 @@ RSpec.describe AccountsController, type: :controller do context 'in authorized fetch mode' do let(:authorized_fetch_mode) { true } - it 'returns http success' do - expect(response).to have_http_status(200) - end - - it 'returns application/activity+json' do - expect(response.content_type).to eq 'application/activity+json' - end - - it_behaves_like 'cachable response' - - it 'returns Vary header with Signature' do - expect(response.headers['Vary']).to include 'Signature' - end - - it 'renders bare minimum account' do - json = body_as_json - expect(json).to include(:id, :type, :preferredUsername, :inbox, :publicKey) - expect(json).to_not include(:name, :summary) + it 'returns http unauthorized' do + expect(response).to have_http_status(401) end end end From 0abfa06b2f4d57363be8690aaf8e8ca3e1bfb221 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 30 Aug 2020 12:33:59 +0200 Subject: [PATCH 02/23] Fix inefficiencies in fan-out-on-write service (#14682) --- app/services/fan_out_on_write_service.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb index 276eac0c1..21931c2f1 100644 --- a/app/services/fan_out_on_write_service.rb +++ b/app/services/fan_out_on_write_service.rb @@ -6,8 +6,6 @@ class FanOutOnWriteService < BaseService def call(status) raise Mastodon::RaceConditionError if status.visibility.nil? - render_anonymous_payload(status) - if status.direct_visibility? deliver_to_own_conversation(status) elsif status.limited_visibility? @@ -20,6 +18,8 @@ class FanOutOnWriteService < BaseService return if status.account.silenced? || !status.public_visibility? || status.reblog? + render_anonymous_payload(status) + deliver_to_hashtags(status) return if status.reply? && status.in_reply_to_account_id != status.account_id @@ -58,8 +58,10 @@ class FanOutOnWriteService < BaseService def deliver_to_mentioned_followers(status) Rails.logger.debug "Delivering status #{status.id} to limited followers" - FeedInsertWorker.push_bulk(status.mentions.includes(:account).map(&:account).select { |mentioned_account| mentioned_account.local? && mentioned_account.following?(status.account) }) do |follower| - [status.id, follower.id, :home] + status.mentions.joins(:account).merge(status.account.followers_for_local_distribution).select(:id).reorder(nil).find_in_batches do |followers| + FeedInsertWorker.push_bulk(followers) do |follower| + [status.id, follower.id, :home] + end end end From c98b7751ca6f7c638997c26b0807af5b51915593 Mon Sep 17 00:00:00 2001 From: Takeshi Umeda Date: Tue, 1 Sep 2020 01:11:27 +0900 Subject: [PATCH 03/23] Fix limited follower id in fan-out-on-write service (#14709) --- app/services/fan_out_on_write_service.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb index 21931c2f1..e05d02cef 100644 --- a/app/services/fan_out_on_write_service.rb +++ b/app/services/fan_out_on_write_service.rb @@ -58,9 +58,9 @@ class FanOutOnWriteService < BaseService def deliver_to_mentioned_followers(status) Rails.logger.debug "Delivering status #{status.id} to limited followers" - status.mentions.joins(:account).merge(status.account.followers_for_local_distribution).select(:id).reorder(nil).find_in_batches do |followers| - FeedInsertWorker.push_bulk(followers) do |follower| - [status.id, follower.id, :home] + status.mentions.joins(:account).merge(status.account.followers_for_local_distribution).select(:id, :account_id).reorder(nil).find_in_batches do |mentions| + FeedInsertWorker.push_bulk(mentions) do |mention| + [status.id, mention.account_id, :home] end end end From 4acfc3ce83a0f7492137ef0a3b0c78cce0773e6e Mon Sep 17 00:00:00 2001 From: ThibG Date: Sat, 1 Aug 2020 18:20:37 +0200 Subject: [PATCH 04/23] Fix handling of Reject Follow when a matching follow relationship exists (#14479) * Add tests * Fix handling of Reject Follow when a matching follow relationship exists Regression from #12199 --- app/lib/activitypub/activity/reject.rb | 2 +- spec/lib/activitypub/activity/reject_spec.rb | 110 ++++++++++++++++--- 2 files changed, 98 insertions(+), 14 deletions(-) diff --git a/app/lib/activitypub/activity/reject.rb b/app/lib/activitypub/activity/reject.rb index 8d771ed81..886dddb23 100644 --- a/app/lib/activitypub/activity/reject.rb +++ b/app/lib/activitypub/activity/reject.rb @@ -4,7 +4,7 @@ class ActivityPub::Activity::Reject < ActivityPub::Activity def perform return reject_follow_for_relay if relay_follow? return follow_request_from_object.reject! unless follow_request_from_object.nil? - return UnfollowService.new.call(follow_from_object.target_account, @account) unless follow_from_object.nil? + return UnfollowService.new.call(follow_from_object.account, @account) unless follow_from_object.nil? case @object['type'] when 'Follow' diff --git a/spec/lib/activitypub/activity/reject_spec.rb b/spec/lib/activitypub/activity/reject_spec.rb index e7205df8d..fed4cd8cd 100644 --- a/spec/lib/activitypub/activity/reject_spec.rb +++ b/spec/lib/activitypub/activity/reject_spec.rb @@ -3,6 +3,14 @@ require 'rails_helper' RSpec.describe ActivityPub::Activity::Reject do let(:sender) { Fabricate(:account) } let(:recipient) { Fabricate(:account) } + let(:object_json) do + { + id: 'bar', + type: 'Follow', + actor: ActivityPub::TagManager.instance.uri_for(recipient), + object: ActivityPub::TagManager.instance.uri_for(sender), + } + end let(:json) do { @@ -10,29 +18,105 @@ RSpec.describe ActivityPub::Activity::Reject do id: 'foo', type: 'Reject', actor: ActivityPub::TagManager.instance.uri_for(sender), - object: { - id: 'bar', - type: 'Follow', - actor: ActivityPub::TagManager.instance.uri_for(recipient), - object: ActivityPub::TagManager.instance.uri_for(sender), - }, + object: object_json, }.with_indifferent_access end describe '#perform' do subject { described_class.new(json, sender) } - before do - Fabricate(:follow_request, account: recipient, target_account: sender) - subject.perform + context 'rejecting a pending follow request by target' do + before do + Fabricate(:follow_request, account: recipient, target_account: sender) + subject.perform + end + + it 'does not create a follow relationship' do + expect(recipient.following?(sender)).to be false + end + + it 'removes the follow request' do + expect(recipient.requested?(sender)).to be false + end end - it 'does not create a follow relationship' do - expect(recipient.following?(sender)).to be false + context 'rejecting a pending follow request by uri' do + before do + Fabricate(:follow_request, account: recipient, target_account: sender, uri: 'bar') + subject.perform + end + + it 'does not create a follow relationship' do + expect(recipient.following?(sender)).to be false + end + + it 'removes the follow request' do + expect(recipient.requested?(sender)).to be false + end end - it 'removes the follow request' do - expect(recipient.requested?(sender)).to be false + context 'rejecting a pending follow request by uri only' do + let(:object_json) { 'bar' } + + before do + Fabricate(:follow_request, account: recipient, target_account: sender, uri: 'bar') + subject.perform + end + + it 'does not create a follow relationship' do + expect(recipient.following?(sender)).to be false + end + + it 'removes the follow request' do + expect(recipient.requested?(sender)).to be false + end + end + + context 'rejecting an existing follow relationship by target' do + before do + Fabricate(:follow, account: recipient, target_account: sender) + subject.perform + end + + it 'removes the follow relationship' do + expect(recipient.following?(sender)).to be false + end + + it 'does not create a follow request' do + expect(recipient.requested?(sender)).to be false + end + end + + context 'rejecting an existing follow relationship by uri' do + before do + Fabricate(:follow, account: recipient, target_account: sender, uri: 'bar') + subject.perform + end + + it 'removes the follow relationship' do + expect(recipient.following?(sender)).to be false + end + + it 'does not create a follow request' do + expect(recipient.requested?(sender)).to be false + end + end + + context 'rejecting an existing follow relationship by uri only' do + let(:object_json) { 'bar' } + + before do + Fabricate(:follow, account: recipient, target_account: sender, uri: 'bar') + subject.perform + end + + it 'removes the follow relationship' do + expect(recipient.following?(sender)).to be false + end + + it 'does not create a follow request' do + expect(recipient.requested?(sender)).to be false + end end end From 8f79ed0487fb17ad59182b49b3fbe46043cbaedd Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 13 Sep 2020 12:52:17 +0200 Subject: [PATCH 05/23] Fix reported statuses not being included in warning e-mail (#14778) --- app/models/admin/account_action.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/admin/account_action.rb b/app/models/admin/account_action.rb index b30a82369..9edd152f5 100644 --- a/app/models/admin/account_action.rb +++ b/app/models/admin/account_action.rb @@ -142,7 +142,7 @@ class Admin::AccountAction end def status_ids - @report.status_ids if @report && include_statuses + report.status_ids if report && include_statuses end def reports From ce6aaed4325d1a5dc15a799856d26b3d22222633 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 8 Oct 2020 00:34:57 +0200 Subject: [PATCH 06/23] Remove dependency on goldfinger gem (#14919) There are edge cases where requests to certain hosts timeout when using the vanilla HTTP.rb gem, which the goldfinger gem uses. Now that we no longer need to support OStatus servers, webfinger logic is so simple that there is no point encapsulating it in a gem, so we can just use our own Request class. With that, we benefit from more robust timeout code and IPv4/IPv6 resolution. Fix #14091 --- Gemfile | 1 - Gemfile.lock | 6 -- app/helpers/webfinger_helper.rb | 33 +------ app/lib/webfinger.rb | 93 +++++++++++++++++++ app/models/account_alias.rb | 2 +- app/models/account_migration.rb | 2 +- app/models/form/redirect.rb | 2 +- app/models/remote_follow.rb | 10 +- .../fetch_remote_account_service.rb | 7 +- app/services/process_mentions_service.rb | 2 +- app/services/resolve_account_service.rb | 9 +- app/views/well_known/host_meta/show.xml.ruby | 1 - .../remote_follow_controller_spec.rb | 10 +- .../well_known/host_meta_controller_spec.rb | 2 +- spec/lib/feed_manager_spec.rb | 1 + 15 files changed, 114 insertions(+), 67 deletions(-) create mode 100644 app/lib/webfinger.rb diff --git a/Gemfile b/Gemfile index 414bd2c30..0891371be 100644 --- a/Gemfile +++ b/Gemfile @@ -54,7 +54,6 @@ gem 'doorkeeper', '~> 5.4' gem 'ed25519', '~> 1.2' gem 'fast_blank', '~> 1.0' gem 'fastimage' -gem 'goldfinger', '~> 2.1' gem 'hiredis', '~> 0.6' gem 'redis-namespace', '~> 1.7' gem 'health_check', git: 'https://github.com/ianheggie/health_check', ref: '0b799ead604f900ed50685e9b2d469cd2befba5b' diff --git a/Gemfile.lock b/Gemfile.lock index 3d4fce643..62c20bf07 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -250,11 +250,6 @@ GEM ruby-progressbar (~> 1.4) globalid (0.4.2) activesupport (>= 4.2.0) - goldfinger (2.1.1) - addressable (~> 2.5) - http (~> 4.0) - nokogiri (~> 1.8) - oj (~> 3.0) hamlit (2.11.0) temple (>= 0.8.2) thor @@ -708,7 +703,6 @@ DEPENDENCIES fog-core (<= 2.1.0) fog-openstack (~> 0.3) fuubar (~> 2.5) - goldfinger (~> 2.1) hamlit-rails (~> 0.2) health_check! hiredis (~> 0.6) diff --git a/app/helpers/webfinger_helper.rb b/app/helpers/webfinger_helper.rb index ab7ca4698..482f4e19e 100644 --- a/app/helpers/webfinger_helper.rb +++ b/app/helpers/webfinger_helper.rb @@ -1,38 +1,7 @@ # frozen_string_literal: true -# Monkey-patch on monkey-patch. -# Because it conflicts with the request.rb patch. -class HTTP::Timeout::PerOperationOriginal < HTTP::Timeout::PerOperation - def connect(socket_class, host, port, nodelay = false) - ::Timeout.timeout(@connect_timeout, HTTP::TimeoutError) do - @socket = socket_class.open(host, port) - @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay - end - end -end - module WebfingerHelper def webfinger!(uri) - hidden_service_uri = /\.(onion|i2p)(:\d+)?$/.match(uri) - - raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if !Rails.configuration.x.access_to_hidden_service && hidden_service_uri - - opts = { - ssl: !hidden_service_uri, - - headers: { - 'User-Agent': Mastodon::Version.user_agent, - }, - - timeout_class: HTTP::Timeout::PerOperationOriginal, - - timeout_options: { - write_timeout: 10, - connect_timeout: 5, - read_timeout: 10, - }, - } - - Goldfinger::Client.new(uri, opts.merge(Rails.configuration.x.http_client_proxy)).finger + Webfinger.new(uri).perform end end diff --git a/app/lib/webfinger.rb b/app/lib/webfinger.rb new file mode 100644 index 000000000..b2374c494 --- /dev/null +++ b/app/lib/webfinger.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +class Webfinger + class Error < StandardError; end + + class Response + def initialize(body) + @json = Oj.load(body, mode: :strict) + end + + def subject + @json['subject'] + end + + def link(rel, attribute) + links.dig(rel, attribute) + end + + private + + def links + @links ||= @json['links'].map { |link| [link['rel'], link] }.to_h + end + end + + def initialize(uri) + _, @domain = uri.split('@') + + raise ArgumentError, 'Webfinger requested for local account' if @domain.nil? + + @uri = uri + end + + def perform + Response.new(body_from_webfinger) + rescue Oj::ParseError + raise Webfinger::Error, "Invalid JSON in response for #{@uri}" + rescue Addressable::URI::InvalidURIError + raise Webfinger::Error, "Invalid URI for #{@uri}" + end + + private + + def body_from_webfinger(url = standard_url, use_fallback = true) + webfinger_request(url).perform do |res| + if res.code == 200 + res.body_with_limit + elsif res.code == 404 && use_fallback + body_from_host_meta + else + raise Webfinger::Error, "Request for #{@uri} returned HTTP #{res.code}" + end + end + end + + def body_from_host_meta + host_meta_request.perform do |res| + if res.code == 200 + body_from_webfinger(url_from_template(res.body_with_limit), false) + else + raise Webfinger::Error, "Request for #{@uri} returned HTTP #{res.code}" + end + end + end + + def url_from_template(str) + link = Nokogiri::XML(str).at_xpath('//xmlns:Link[@rel="lrdd"]') + + if link.present? + link['template'].gsub('{uri}', @uri) + else + raise Webfinger::Error, "Request for #{@uri} returned host-meta without link to Webfinger" + end + rescue Nokogiri::XML::XPath::SyntaxError + raise Webfinger::Error, "Invalid XML encountered in host-meta for #{@uri}" + end + + def host_meta_request + Request.new(:get, host_meta_url).add_headers('Accept' => 'application/xrd+xml, application/xml, text/xml') + end + + def webfinger_request(url) + Request.new(:get, url).add_headers('Accept' => 'application/jrd+json, application/json') + end + + def standard_url + "https://#{@domain}/.well-known/webfinger?resource=#{@uri}" + end + + def host_meta_url + "https://#{@domain}/.well-known/host-meta" + end +end diff --git a/app/models/account_alias.rb b/app/models/account_alias.rb index 792e9e8d4..3d659142a 100644 --- a/app/models/account_alias.rb +++ b/app/models/account_alias.rb @@ -33,7 +33,7 @@ class AccountAlias < ApplicationRecord def set_uri target_account = ResolveAccountService.new.call(acct) self.uri = ActivityPub::TagManager.instance.uri_for(target_account) unless target_account.nil? - rescue Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error + rescue Webfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error # Validation will take care of it end diff --git a/app/models/account_migration.rb b/app/models/account_migration.rb index 681b5b2cd..4fae98ed7 100644 --- a/app/models/account_migration.rb +++ b/app/models/account_migration.rb @@ -54,7 +54,7 @@ class AccountMigration < ApplicationRecord def set_target_account self.target_account = ResolveAccountService.new.call(acct) - rescue Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error + rescue Webfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error # Validation will take care of it end diff --git a/app/models/form/redirect.rb b/app/models/form/redirect.rb index a7961f8e8..19ee9faed 100644 --- a/app/models/form/redirect.rb +++ b/app/models/form/redirect.rb @@ -32,7 +32,7 @@ class Form::Redirect def set_target_account @target_account = ResolveAccountService.new.call(acct) - rescue Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error + rescue Webfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error # Validation will take care of it end diff --git a/app/models/remote_follow.rb b/app/models/remote_follow.rb index 30b84f7d5..911c06713 100644 --- a/app/models/remote_follow.rb +++ b/app/models/remote_follow.rb @@ -56,7 +56,7 @@ class RemoteFollow if domain.nil? @addressable_template = Addressable::Template.new("#{authorize_interaction_url}?uri={uri}") - elsif redirect_url_link.nil? || redirect_url_link.template.nil? + elsif redirect_uri_template.nil? missing_resource_error else @addressable_template = Addressable::Template.new(redirect_uri_template) @@ -64,16 +64,12 @@ class RemoteFollow end def redirect_uri_template - redirect_url_link.template - end - - def redirect_url_link - acct_resource&.link('http://ostatus.org/schema/1.0/subscribe') + acct_resource&.link('http://ostatus.org/schema/1.0/subscribe', 'template') end def acct_resource @acct_resource ||= webfinger!("acct:#{acct}") - rescue Goldfinger::Error, HTTP::ConnectionError + rescue Webfinger::Error, HTTP::ConnectionError nil end diff --git a/app/services/activitypub/fetch_remote_account_service.rb b/app/services/activitypub/fetch_remote_account_service.rb index 83fbf6d07..e5bd0c47c 100644 --- a/app/services/activitypub/fetch_remote_account_service.rb +++ b/app/services/activitypub/fetch_remote_account_service.rb @@ -39,17 +39,16 @@ class ActivityPub::FetchRemoteAccountService < BaseService webfinger = webfinger!("acct:#{@username}@#{@domain}") confirmed_username, confirmed_domain = split_acct(webfinger.subject) - return webfinger.link('self')&.href == @uri if @username.casecmp(confirmed_username).zero? && @domain.casecmp(confirmed_domain).zero? + return webfinger.link('self', 'href') == @uri if @username.casecmp(confirmed_username).zero? && @domain.casecmp(confirmed_domain).zero? webfinger = webfinger!("acct:#{confirmed_username}@#{confirmed_domain}") @username, @domain = split_acct(webfinger.subject) - self_reference = webfinger.link('self') return false unless @username.casecmp(confirmed_username).zero? && @domain.casecmp(confirmed_domain).zero? - return false if self_reference&.href != @uri + return false if webfinger.link('self', 'href') != @uri true - rescue Goldfinger::Error + rescue Webfinger::Error false end diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb index 79af3fc54..8e260811d 100644 --- a/app/services/process_mentions_service.rb +++ b/app/services/process_mentions_service.rb @@ -29,7 +29,7 @@ class ProcessMentionsService < BaseService if mention_undeliverable?(mentioned_account) begin mentioned_account = resolve_account_service.call(Regexp.last_match(1)) - rescue Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::UnexpectedResponseError + rescue Webfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::UnexpectedResponseError mentioned_account = nil end end diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb index ba77552c6..3f7bb7cc5 100644 --- a/app/services/resolve_account_service.rb +++ b/app/services/resolve_account_service.rb @@ -26,11 +26,10 @@ class ResolveAccountService < BaseService @account ||= Account.find_remote(@username, @domain) - return @account if @account&.local? || !webfinger_update_due? + return @account if @account&.local? || @domain.nil? || !webfinger_update_due? # At this point we are in need of a Webfinger query, which may # yield us a different username/domain through a redirect - process_webfinger!(@uri) # Because the username/domain pair may be different than what @@ -47,7 +46,7 @@ class ResolveAccountService < BaseService # either needs to be created, or updated from fresh data process_account! - rescue Goldfinger::Error, WebfingerRedirectError, Oj::ParseError => e + rescue Webfinger::Error, WebfingerRedirectError, Oj::ParseError => e Rails.logger.debug "Webfinger query for #{@uri} failed: #{e}" nil end @@ -118,11 +117,11 @@ class ResolveAccountService < BaseService end def activitypub_ready? - !@webfinger.link('self').nil? && ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@webfinger.link('self').type) + ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@webfinger.link('self', 'type')) end def actor_url - @actor_url ||= @webfinger.link('self').href + @actor_url ||= @webfinger.link('self', 'href') end def actor_json diff --git a/app/views/well_known/host_meta/show.xml.ruby b/app/views/well_known/host_meta/show.xml.ruby index 0a6bdc322..b4e867c5f 100644 --- a/app/views/well_known/host_meta/show.xml.ruby +++ b/app/views/well_known/host_meta/show.xml.ruby @@ -5,7 +5,6 @@ doc << Ox::Element.new('XRD').tap do |xrd| xrd << Ox::Element.new('Link').tap do |link| link['rel'] = 'lrdd' - link['type'] = 'application/xrd+xml' link['template'] = @webfinger_template end end diff --git a/spec/controllers/remote_follow_controller_spec.rb b/spec/controllers/remote_follow_controller_spec.rb index 3ef8f14d9..7312dde58 100644 --- a/spec/controllers/remote_follow_controller_spec.rb +++ b/spec/controllers/remote_follow_controller_spec.rb @@ -43,8 +43,7 @@ describe RemoteFollowController do end it 'renders new when template is nil' do - link_with_nil_template = double(template: nil) - resource_with_link = double(link: link_with_nil_template) + resource_with_link = double(link: nil) allow_any_instance_of(WebfingerHelper).to receive(:webfinger!).with('acct:user@example.com').and_return(resource_with_link) post :create, params: { account_username: @account.to_param, remote_follow: { acct: 'user@example.com' } } @@ -55,8 +54,7 @@ describe RemoteFollowController do context 'when webfinger values are good' do before do - link_with_template = double(template: 'http://example.com/follow_me?acct={uri}') - resource_with_link = double(link: link_with_template) + resource_with_link = double(link: 'http://example.com/follow_me?acct={uri}') allow_any_instance_of(WebfingerHelper).to receive(:webfinger!).with('acct:user@example.com').and_return(resource_with_link) post :create, params: { account_username: @account.to_param, remote_follow: { acct: 'user@example.com' } } end @@ -78,8 +76,8 @@ describe RemoteFollowController do expect(response).to render_template(:new) end - it 'renders new with error when goldfinger fails' do - allow_any_instance_of(WebfingerHelper).to receive(:webfinger!).with('acct:user@example.com').and_raise(Goldfinger::Error) + it 'renders new with error when webfinger fails' do + allow_any_instance_of(WebfingerHelper).to receive(:webfinger!).with('acct:user@example.com').and_raise(Webfinger::Error) post :create, params: { account_username: @account.to_param, remote_follow: { acct: 'user@example.com' } } expect(response).to render_template(:new) diff --git a/spec/controllers/well_known/host_meta_controller_spec.rb b/spec/controllers/well_known/host_meta_controller_spec.rb index b43ae19d8..643ba9cd3 100644 --- a/spec/controllers/well_known/host_meta_controller_spec.rb +++ b/spec/controllers/well_known/host_meta_controller_spec.rb @@ -12,7 +12,7 @@ describe WellKnown::HostMetaController, type: :controller do expect(response.body).to eq < - + XML end diff --git a/spec/lib/feed_manager_spec.rb b/spec/lib/feed_manager_spec.rb index 22eddd2ab..2d1f6cf96 100644 --- a/spec/lib/feed_manager_spec.rb +++ b/spec/lib/feed_manager_spec.rb @@ -108,6 +108,7 @@ RSpec.describe FeedManager do it 'returns false for status by followee mentioning another account' do bob.follow!(alice) + jeff.follow!(alice) status = PostStatusService.new.call(alice, text: 'Hey @jeff') expect(FeedManager.instance.filter?(:home, status, bob.id)).to be false end From 3f4cceebd66c0e209239bf5a917bbda8de57d189 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 30 Aug 2020 01:54:30 +0200 Subject: [PATCH 07/23] Fix videos with near-60 fps being rejected (#14684) Fix #14668 --- app/models/media_attachment.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 3d93ec75b..663bb0896 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -338,7 +338,7 @@ class MediaAttachment < ApplicationRecord raise Mastodon::StreamValidationError, 'Video has no video stream' if movie.width.nil? || movie.frame_rate.nil? raise Mastodon::DimensionsValidationError, "#{movie.width}x#{movie.height} videos are not supported" if movie.width * movie.height > MAX_VIDEO_MATRIX_LIMIT - raise Mastodon::DimensionsValidationError, "#{movie.frame_rate.to_i}fps videos are not supported" if movie.frame_rate > MAX_VIDEO_FRAME_RATE + raise Mastodon::DimensionsValidationError, "#{movie.frame_rate.floor}fps videos are not supported" if movie.frame_rate.floor > MAX_VIDEO_FRAME_RATE end def set_meta From 58c59af573d7cb285317bdb27d745b38cf045378 Mon Sep 17 00:00:00 2001 From: Takeshi Umeda Date: Tue, 25 Aug 2020 01:09:46 +0900 Subject: [PATCH 08/23] Fix an error when file_file_size is nil in tootctl media remove (#14657) --- lib/mastodon/media_cli.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index 2a4e3e379..45181f59e 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -31,7 +31,7 @@ module Mastodon processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where('created_at < ?', time_ago)) do |media_attachment| next if media_attachment.file.blank? - size = media_attachment.file_file_size + (media_attachment.thumbnail_file_size || 0) + size = (media_attachment.file_file_size || 0) + (media_attachment.thumbnail_file_size || 0) unless options[:dry_run] media_attachment.file.destroy From 856cb96a2b4823b62df19f67686921890adfc2f8 Mon Sep 17 00:00:00 2001 From: ThibG Date: Sun, 2 Aug 2020 11:20:17 +0200 Subject: [PATCH 09/23] Fix new audio player features not working on Safari (#14465) Fixes #14462 --- app/javascript/mastodon/features/audio/index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/javascript/mastodon/features/audio/index.js b/app/javascript/mastodon/features/audio/index.js index 1ab1c3117..a4e00ba96 100644 --- a/app/javascript/mastodon/features/audio/index.js +++ b/app/javascript/mastodon/features/audio/index.js @@ -269,8 +269,9 @@ class Audio extends React.PureComponent { } _initAudioContext () { - const context = new AudioContext(); - const source = context.createMediaElementSource(this.audio); + const AudioContext = window.AudioContext || window.webkitAudioContext; + const context = new AudioContext(); + const source = context.createMediaElementSource(this.audio); this.visualizer.setAudioContext(context, source); source.connect(context.destination); From 399c5f09009e05d22e9acd8bb75f3f803b58e365 Mon Sep 17 00:00:00 2001 From: ThibG Date: Sun, 2 Aug 2020 11:21:10 +0200 Subject: [PATCH 10/23] Change content-type to be always computed from file data (#14452) * Change content-type to be always computed from file data Restore previous behavior, detecting the content-type isn't very expensive, and some instances may serve files as application/octet-stream regardless of their true type, making fetching media from them fail, while it used to work pre-3.2.0. * Add test --- lib/paperclip/response_with_limit_adapter.rb | 2 +- spec/lib/activitypub/activity/create_spec.rb | 27 ++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/lib/paperclip/response_with_limit_adapter.rb b/lib/paperclip/response_with_limit_adapter.rb index 7d897b8d6..8711b1349 100644 --- a/lib/paperclip/response_with_limit_adapter.rb +++ b/lib/paperclip/response_with_limit_adapter.rb @@ -19,7 +19,7 @@ module Paperclip @original_filename = filename_from_content_disposition || filename_from_path || 'data' @size = @target.response.content_length @tempfile = copy_to_tempfile(@target) - @content_type = @target.response.mime_type || ContentTypeDetector.new(@tempfile.path).detect + @content_type = ContentTypeDetector.new(@tempfile.path).detect end def copy_to_tempfile(source) diff --git a/spec/lib/activitypub/activity/create_spec.rb b/spec/lib/activitypub/activity/create_spec.rb index 2ac4acc12..51e0b8caf 100644 --- a/spec/lib/activitypub/activity/create_spec.rb +++ b/spec/lib/activitypub/activity/create_spec.rb @@ -18,6 +18,7 @@ RSpec.describe ActivityPub::Activity::Create do stub_request(:get, 'http://example.com/attachment.png').to_return(request_fixture('avatar.txt')) stub_request(:get, 'http://example.com/emoji.png').to_return(body: attachment_fixture('emojo.png')) + stub_request(:get, 'http://example.com/emojib.png').to_return(body: attachment_fixture('emojo.png'), headers: { 'Content-Type' => 'application/octet-stream' }) end describe '#perform' do @@ -451,6 +452,32 @@ RSpec.describe ActivityPub::Activity::Create do end end + context 'with emojis served with invalid content-type' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum :tinkong:', + tag: [ + { + type: 'Emoji', + icon: { + url: 'http://example.com/emojib.png', + }, + name: 'tinkong', + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.emojis.map(&:shortcode)).to include('tinkong') + end + end + context 'with emojis missing name' do let(:object_json) do { From 469c4c78a3ce2f7065c7273fd2800f9a39191a21 Mon Sep 17 00:00:00 2001 From: ThibG Date: Sun, 2 Aug 2020 18:47:09 +0200 Subject: [PATCH 11/23] Fix audio player on Safari (#14485) --- app/javascript/mastodon/features/audio/index.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/javascript/mastodon/features/audio/index.js b/app/javascript/mastodon/features/audio/index.js index a4e00ba96..5b8172694 100644 --- a/app/javascript/mastodon/features/audio/index.js +++ b/app/javascript/mastodon/features/audio/index.js @@ -115,6 +115,10 @@ class Audio extends React.PureComponent { } togglePlay = () => { + if (!this.audioContext) { + this._initAudioContext(); + } + if (this.state.paused) { this.setState({ paused: false }, () => this.audio.play()); } else { @@ -133,10 +137,6 @@ class Audio extends React.PureComponent { handlePlay = () => { this.setState({ paused: false }); - if (this.canvas && !this.audioContext) { - this._initAudioContext(); - } - if (this.audioContext && this.audioContext.state === 'suspended') { this.audioContext.resume(); } From 1995a5cb34337d18ba305c56715194fbaa68786e Mon Sep 17 00:00:00 2001 From: ThibG Date: Sun, 2 Aug 2020 19:03:10 +0200 Subject: [PATCH 12/23] Fix audio/video player not using CDN_HOST in media paths on public pages (#14486) --- app/views/statuses/_detailed_status.html.haml | 4 ++-- app/views/statuses/_simple_status.html.haml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/views/statuses/_detailed_status.html.haml b/app/views/statuses/_detailed_status.html.haml index 85b2ceea4..b3e9c44fc 100644 --- a/app/views/statuses/_detailed_status.html.haml +++ b/app/views/statuses/_detailed_status.html.haml @@ -29,11 +29,11 @@ - if !status.media_attachments.empty? - if status.media_attachments.first.video? - video = status.media_attachments.first - = react_component :video, src: video.file.url(:original), preview: video.thumbnail.present? ? video.thumbnail.url : video.file.url(:small), blurhash: video.blurhash, sensitive: status.sensitive?, width: 670, height: 380, detailed: true, inline: true, alt: video.description do + = react_component :video, src: full_asset_url(video.file.url(:original)), preview: full_asset_url(video.thumbnail.present? ? video.thumbnail.url : video.file.url(:small)), blurhash: video.blurhash, sensitive: status.sensitive?, width: 670, height: 380, detailed: true, inline: true, alt: video.description do = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } - elsif status.media_attachments.first.audio? - audio = status.media_attachments.first - = react_component :audio, src: audio.file.url(:original), poster: audio.thumbnail.present? ? audio.thumbnail.url : status.account.avatar_static_url, backgroundColor: audio.file.meta.dig('colors', 'background'), foregroundColor: audio.file.meta.dig('colors', 'foreground'), accentColor: audio.file.meta.dig('colors', 'accent'), width: 670, height: 380, alt: audio.description, duration: audio.file.meta.dig('original', 'duration') do + = react_component :audio, src: full_asset_url(audio.file.url(:original)), poster: full_asset_url(audio.thumbnail.present? ? audio.thumbnail.url : status.account.avatar_static_url), backgroundColor: audio.file.meta.dig('colors', 'background'), foregroundColor: audio.file.meta.dig('colors', 'foreground'), accentColor: audio.file.meta.dig('colors', 'accent'), width: 670, height: 380, alt: audio.description, duration: audio.file.meta.dig('original', 'duration') do = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } - else = react_component :media_gallery, height: 380, sensitive: status.sensitive?, standalone: true, autoplay: autoplay, media: status.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } do diff --git a/app/views/statuses/_simple_status.html.haml b/app/views/statuses/_simple_status.html.haml index 67c6c0fd0..363757945 100644 --- a/app/views/statuses/_simple_status.html.haml +++ b/app/views/statuses/_simple_status.html.haml @@ -35,11 +35,11 @@ - if !status.media_attachments.empty? - if status.media_attachments.first.video? - video = status.media_attachments.first - = react_component :video, src: video.file.url(:original), preview: video.thumbnail.present? ? video.thumbnail.url : video.file.url(:small), blurhash: video.blurhash, sensitive: status.sensitive?, width: 610, height: 343, inline: true, alt: video.description do + = react_component :video, src: full_asset_url(video.file.url(:original)), preview: full_asset_url(video.thumbnail.present? ? video.thumbnail.url : video.file.url(:small)), blurhash: video.blurhash, sensitive: status.sensitive?, width: 610, height: 343, inline: true, alt: video.description do = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } - elsif status.media_attachments.first.audio? - audio = status.media_attachments.first - = react_component :audio, src: audio.file.url(:original), poster: audio.thumbnail.present? ? audio.thumbnail.url : status.account.avatar_static_url, backgroundColor: audio.file.meta.dig('colors', 'background'), foregroundColor: audio.file.meta.dig('colors', 'foreground'), accentColor: audio.file.meta.dig('colors', 'accent'), width: 610, height: 343, alt: audio.description, duration: audio.file.meta.dig('original', 'duration') do + = react_component :audio, src: full_asset_url(audio.file.url(:original)), poster: full_asset_url(audio.thumbnail.present? ? audio.thumbnail.url : status.account.avatar_static_url), backgroundColor: audio.file.meta.dig('colors', 'background'), foregroundColor: audio.file.meta.dig('colors', 'foreground'), accentColor: audio.file.meta.dig('colors', 'accent'), width: 610, height: 343, alt: audio.description, duration: audio.file.meta.dig('original', 'duration') do = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } - else = react_component :media_gallery, height: 343, sensitive: status.sensitive?, autoplay: autoplay, media: status.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } do From 3b699f17320de7fc1d1adc40e8edbd8ee58c9d57 Mon Sep 17 00:00:00 2001 From: ThibG Date: Sun, 2 Aug 2020 18:47:44 +0200 Subject: [PATCH 13/23] Fix thumbnail color extraction (#14464) * Fix contrast calculation for thumbnail color extraction Luminance calculation was using 0-255 RGB values instead of 0-1 sRGB values, leading to incorrectly-computed contrast values. Since we use ColorDiff already, just use its XYZ colorspace conversion code to get the value. * Require at least 3:1 contrast for both accent and foreground colors * Lower required contrast for the accent color --- lib/paperclip/color_extractor.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/paperclip/color_extractor.rb b/lib/paperclip/color_extractor.rb index 44fe5ff1d..c8bb771a0 100644 --- a/lib/paperclip/color_extractor.rb +++ b/lib/paperclip/color_extractor.rb @@ -5,6 +5,7 @@ require 'mime/types/columnar' module Paperclip class ColorExtractor < Paperclip::Processor MIN_CONTRAST = 3.0 + ACCENT_MIN_CONTRAST = 2.0 FREQUENCY_THRESHOLD = 0.01 def make @@ -26,8 +27,9 @@ module Paperclip foreground_palette.each do |color| distance = ColorDiff.between(background_color, color) + contrast = w3c_contrast(background_color, color) - if distance > max_distance + if distance > max_distance && contrast >= ACCENT_MIN_CONTRAST max_distance = distance max_distance_color = color end @@ -77,8 +79,8 @@ module Paperclip private def w3c_contrast(color1, color2) - luminance1 = (0.2126 * color1.r + 0.7152 * color1.g + 0.0722 * color1.b) + 0.05 - luminance2 = (0.2126 * color2.r + 0.7152 * color2.g + 0.0722 * color2.b) + 0.05 + luminance1 = color1.to_xyz.y * 0.01 + 0.05 + luminance2 = color2.to_xyz.y * 0.01 + 0.05 if luminance1 > luminance2 luminance1 / luminance2 From 6db143e424b7566519153e6a0c831cd77ceff227 Mon Sep 17 00:00:00 2001 From: ThibG Date: Sat, 8 Aug 2020 17:57:56 +0200 Subject: [PATCH 14/23] Fix crash when failing to load emoji picker (#14525) Fixes #14523 --- .../features/compose/components/emoji_picker_dropdown.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js b/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js index 360a7af6a..e8a36a923 100644 --- a/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js +++ b/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js @@ -315,7 +315,7 @@ class EmojiPickerDropdown extends React.PureComponent { this.setState({ loading: false }); }).catch(() => { - this.setState({ loading: false }); + this.setState({ loading: false, active: false }); }); } From 8b448aecef9495353a1cd18d9e5d95b576cdede2 Mon Sep 17 00:00:00 2001 From: ThibG Date: Mon, 10 Aug 2020 01:51:06 +0200 Subject: [PATCH 15/23] Fix `tootctl media` commands not handling snowflake ids for media_attachments (#14536) --- lib/mastodon/media_cli.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index 45181f59e..54da5b2cd 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -89,7 +89,7 @@ module Mastodon path_segments = object.key.split('/') path_segments.delete('cache') - if path_segments.size != 7 + unless [7, 10].include?(path_segments.size) progress.log(pastel.yellow("Unrecognized file found: #{object.key}")) next end @@ -133,7 +133,7 @@ module Mastodon path_segments = key.split(File::SEPARATOR) path_segments.delete('cache') - if path_segments.size != 7 + unless [7, 10].include?(path_segments.size) progress.log(pastel.yellow("Unrecognized file found: #{key}")) next end @@ -258,7 +258,7 @@ module Mastodon path_segments = path.split('/')[2..-1] path_segments.delete('cache') - if path_segments.size != 7 + unless [7, 10].include?(path_segments.size) say('Not a media URL', :red) exit(1) end @@ -311,7 +311,7 @@ module Mastodon segments = object.key.split('/') segments.delete('cache') - next if segments.size != 7 + next unless [7, 10].include?(segments.size) model_name = segments.first.classify record_id = segments[2..-2].join.to_i From aea0161e83ba0d154a3b3824e4d14d31773486b0 Mon Sep 17 00:00:00 2001 From: ThibG Date: Mon, 24 Aug 2020 14:11:47 +0200 Subject: [PATCH 16/23] Add support for inlined objects in activity audience (#14514) * Add support for inlined objects in activity audience * Add tests --- app/lib/activitypub/activity.rb | 2 +- app/lib/activitypub/activity/announce.rb | 14 +++++++++--- app/lib/activitypub/activity/create.rb | 16 +++++++------- .../lib/activitypub/activity/announce_spec.rb | 20 +++++++++++++++++ spec/lib/activitypub/activity/create_spec.rb | 22 +++++++++++++++++++ 5 files changed, 62 insertions(+), 12 deletions(-) diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index ab946470b..f0ef4d553 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -172,7 +172,7 @@ class ActivityPub::Activity end def first_mentioned_local_account - audience = (as_array(@json['to']) + as_array(@json['cc'])).uniq + audience = (as_array(@json['to']) + as_array(@json['cc'])).map { |x| value_or_id(x) }.uniq local_usernames = audience.select { |uri| ActivityPub::TagManager.instance.local_uri?(uri) } .map { |uri| ActivityPub::TagManager.instance.uri_to_local_id(uri, :username) } diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index 9e108985a..349e8f77e 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -34,12 +34,20 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity private + def audience_to + as_array(@json['to']).map { |x| value_or_id(x) } + end + + def audience_cc + as_array(@json['cc']).map { |x| value_or_id(x) } + end + def visibility_from_audience - if equals_or_includes?(@json['to'], ActivityPub::TagManager::COLLECTIONS[:public]) + if audience_to.include?(ActivityPub::TagManager::COLLECTIONS[:public]) :public - elsif equals_or_includes?(@json['cc'], ActivityPub::TagManager::COLLECTIONS[:public]) + elsif audience_cc.include?(ActivityPub::TagManager::COLLECTIONS[:public]) :unlisted - elsif equals_or_includes?(@json['to'], @account.followers_url) + elsif audience_to.include?(@account.followers_url) :private else :direct diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 08dd98e94..a60b79d15 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -65,11 +65,11 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def audience_to - @object['to'] || @json['to'] + as_array(@object['to'] || @json['to']).map { |x| value_or_id(x) } end def audience_cc - @object['cc'] || @json['cc'] + as_array(@object['cc'] || @json['cc']).map { |x| value_or_id(x) } end def process_status @@ -122,7 +122,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def process_audience - (as_array(audience_to) + as_array(audience_cc)).uniq.each do |audience| + (audience_to + audience_cc).uniq.each do |audience| next if audience == ActivityPub::TagManager::COLLECTIONS[:public] # Unlike with tags, there is no point in resolving accounts we don't already @@ -352,11 +352,11 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def visibility_from_audience - if equals_or_includes?(audience_to, ActivityPub::TagManager::COLLECTIONS[:public]) + if audience_to.include?(ActivityPub::TagManager::COLLECTIONS[:public]) :public - elsif equals_or_includes?(audience_cc, ActivityPub::TagManager::COLLECTIONS[:public]) + elsif audience_cc.include?(ActivityPub::TagManager::COLLECTIONS[:public]) :unlisted - elsif equals_or_includes?(audience_to, @account.followers_url) + elsif audience_to.include?(@account.followers_url) :private else :direct @@ -365,7 +365,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def audience_includes?(account) uri = ActivityPub::TagManager.instance.uri_for(account) - equals_or_includes?(audience_to, uri) || equals_or_includes?(audience_cc, uri) + audience_to.include?(uri) || audience_cc.include?(uri) end def replied_to_status @@ -477,7 +477,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def addresses_local_accounts? return true if @options[:delivered_to_account_id] - local_usernames = (as_array(audience_to) + as_array(audience_cc)).uniq.select { |uri| ActivityPub::TagManager.instance.local_uri?(uri) }.map { |uri| ActivityPub::TagManager.instance.uri_to_local_id(uri, :username) } + local_usernames = (audience_to + audience_cc).uniq.select { |uri| ActivityPub::TagManager.instance.local_uri?(uri) }.map { |uri| ActivityPub::TagManager.instance.uri_to_local_id(uri, :username) } return false if local_usernames.empty? diff --git a/spec/lib/activitypub/activity/announce_spec.rb b/spec/lib/activitypub/activity/announce_spec.rb index 60fd96a18..b93fcbe66 100644 --- a/spec/lib/activitypub/activity/announce_spec.rb +++ b/spec/lib/activitypub/activity/announce_spec.rb @@ -73,6 +73,26 @@ RSpec.describe ActivityPub::Activity::Announce do expect(sender.reblogged?(sender.statuses.first)).to be true end end + + context 'self-boost of a previously unknown status with correct attributedTo, inlined Collection in audience' do + let(:object_json) do + { + id: 'https://example.com/actor#bar', + type: 'Note', + content: 'Lorem ipsum', + attributedTo: 'https://example.com/actor', + to: { + 'type': 'OrderedCollection', + 'id': 'http://example.com/followers', + 'first': 'http://example.com/followers?page=true', + } + } + end + + it 'creates a reblog by sender of status' do + expect(sender.reblogged?(sender.statuses.first)).to be true + end + end end context 'when the status belongs to a local user' do diff --git a/spec/lib/activitypub/activity/create_spec.rb b/spec/lib/activitypub/activity/create_spec.rb index 51e0b8caf..d2e9fe33c 100644 --- a/spec/lib/activitypub/activity/create_spec.rb +++ b/spec/lib/activitypub/activity/create_spec.rb @@ -121,6 +121,28 @@ RSpec.describe ActivityPub::Activity::Create do end end + context 'private with inlined Collection in audience' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + to: { + 'type': 'OrderedCollection', + 'id': 'http://example.com/followers', + 'first': 'http://example.com/followers?page=true', + } + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.visibility).to eq 'private' + end + end + context 'limited' do let(:recipient) { Fabricate(:account) } From dd3a86eb04d7445e32df44b66ec34332b78b7902 Mon Sep 17 00:00:00 2001 From: Tdxdxoz Date: Mon, 24 Aug 2020 20:13:44 +0800 Subject: [PATCH 17/23] Fix: also use custom private boost icon for detailed status (#14471) * use custom private boost icon for detail status * only use className --- app/javascript/mastodon/components/status_action_bar.js | 3 ++- .../mastodon/features/status/components/action_bar.js | 3 ++- app/javascript/styles/mastodon/boost.scss | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/javascript/mastodon/components/status_action_bar.js b/app/javascript/mastodon/components/status_action_bar.js index 231c517e9..b47855a7e 100644 --- a/app/javascript/mastodon/components/status_action_bar.js +++ b/app/javascript/mastodon/components/status_action_bar.js @@ -7,6 +7,7 @@ import DropdownMenuContainer from '../containers/dropdown_menu_container'; import { defineMessages, injectIntl } from 'react-intl'; import ImmutablePureComponent from 'react-immutable-pure-component'; import { me, isStaff } from '../initial_state'; +import classNames from 'classnames'; const messages = defineMessages({ delete: { id: 'status.delete', defaultMessage: 'Delete' }, @@ -329,7 +330,7 @@ class StatusActionBar extends ImmutablePureComponent { return (
{obfuscatedCount(status.get('replies_count'))}
- + {shareButton} diff --git a/app/javascript/mastodon/features/status/components/action_bar.js b/app/javascript/mastodon/features/status/components/action_bar.js index 1c5d5ca0c..d0f1c57d0 100644 --- a/app/javascript/mastodon/features/status/components/action_bar.js +++ b/app/javascript/mastodon/features/status/components/action_bar.js @@ -6,6 +6,7 @@ import ImmutablePropTypes from 'react-immutable-proptypes'; import DropdownMenuContainer from '../../../containers/dropdown_menu_container'; import { defineMessages, injectIntl } from 'react-intl'; import { me, isStaff } from '../../../initial_state'; +import classNames from 'classnames'; const messages = defineMessages({ delete: { id: 'status.delete', defaultMessage: 'Delete' }, @@ -273,7 +274,7 @@ class ActionBar extends React.PureComponent { return (
-
+
{shareButton}
diff --git a/app/javascript/styles/mastodon/boost.scss b/app/javascript/styles/mastodon/boost.scss index 3489428f8..0be3533bc 100644 --- a/app/javascript/styles/mastodon/boost.scss +++ b/app/javascript/styles/mastodon/boost.scss @@ -6,7 +6,7 @@ button.icon-button i.fa-retweet { } } -.status-private button.icon-button i.fa-retweet { +button.icon-button.reblogPrivate i.fa-retweet { background-image: url("data:image/svg+xml;utf8,"); &:hover { From aa98655cf61e732fb3cfe7626347b79189f61b77 Mon Sep 17 00:00:00 2001 From: ThibG Date: Mon, 24 Aug 2020 16:56:21 +0200 Subject: [PATCH 18/23] Fix dereferencing remote statuses not using the correct account (#14656) Follow-up to #14359 In the case of limited toots, the receiver may not be explicitly part of the audience. If a specific user's inbox URI was specified, it makes sense to dereference the toot from the corresponding user, instead of trying to find someone in the explicit audience. --- app/lib/activitypub/activity.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index f0ef4d553..a379a7ef4 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -168,6 +168,8 @@ class ActivityPub::Activity end def signed_fetch_account + return Account.find(@options[:delivered_to_account_id]) if @options[:delivered_to_account_id].present? + first_mentioned_local_account || first_local_follower end From 4ea7193f0a65a28886b954e99733cc42e6b9f572 Mon Sep 17 00:00:00 2001 From: ThibG Date: Mon, 24 Aug 2020 18:21:07 +0200 Subject: [PATCH 19/23] Add support for latest HTTP Signatures spec draft (#14556) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add support for latest HTTP Signatures spec draft https://www.ietf.org/id/draft-ietf-httpbis-message-signatures-00.html - add support for the “hs2019” signature algorithm (assumed to be equivalent to RSA-SHA256, since we do not have a mechanism to specify the algorithm within the key metadata yet) - add support for (created) and (expires) pseudo-headers and related signature parameters, when using the hs2019 signature algorithm - adjust default “headers” parameter while being backwards-compatible with previous implementation - change the acceptable time window logic from 12 hours surrounding the “date” header to accepting signatures created up to 1 hour in the future and expiring up to 1 hour in the past (but only allowing expiration dates up to 12 hours after the creation date) This doesn't conform with the current draft, as it doesn't permit accounting for clock skew. This, however, should be addressed in a next version of the draft: https://github.com/httpwg/http-extensions/pull/1235 * Add additional signature requirements * Rewrite signature params parsing using Parslet * Make apparent which signature algorithm Mastodon on verification failure Mastodon uses RSASSA-PKCS1-v1_5, which is not recommended for new applications, and new implementers may thus unknowingly use RSASSA-PSS. * Add workaround for PeerTube's invalid signature header The previous parser allowed incorrect Signature headers, such as those produced by old versions of the `http-signature` node.js package, and seemingly used by PeerTube. This commit adds a workaround for that. * Fix `signature_key_id` raising an exception Previously, parsing failures would result in `signature_key_id` being nil, but the parser changes made that result in an exception. This commit changes the `signature_key_id` method to return `nil` in case of parsing failures. * Move extra HTTP signature helper methods to private methods * Relax (request-target) requirement to (request-target) || digest This lets requests from Plume work without lowering security significantly. --- .../concerns/signature_verification.rb | 163 ++++++++++++------ 1 file changed, 108 insertions(+), 55 deletions(-) diff --git a/app/controllers/concerns/signature_verification.rb b/app/controllers/concerns/signature_verification.rb index 10efbf2e0..18f549de9 100644 --- a/app/controllers/concerns/signature_verification.rb +++ b/app/controllers/concerns/signature_verification.rb @@ -7,6 +7,44 @@ module SignatureVerification include DomainControlHelper + EXPIRATION_WINDOW_LIMIT = 12.hours + CLOCK_SKEW_MARGIN = 1.hour + + class SignatureVerificationError < StandardError; end + + class SignatureParamsParser < Parslet::Parser + rule(:token) { match("[0-9a-zA-Z!#$%&'*+.^_`|~-]").repeat(1).as(:token) } + rule(:quoted_string) { str('"') >> (qdtext | quoted_pair).repeat.as(:quoted_string) >> str('"') } + # qdtext and quoted_pair are not exactly according to spec but meh + rule(:qdtext) { match('[^\\\\"]') } + rule(:quoted_pair) { str('\\') >> any } + rule(:bws) { match('\s').repeat } + rule(:param) { (token.as(:key) >> bws >> str('=') >> bws >> (token | quoted_string).as(:value)).as(:param) } + rule(:comma) { bws >> str(',') >> bws } + # Old versions of node-http-signature add an incorrect "Signature " prefix to the header + rule(:buggy_prefix) { str('Signature ') } + rule(:params) { buggy_prefix.maybe >> (param >> (comma >> param).repeat).as(:params) } + root(:params) + end + + class SignatureParamsTransformer < Parslet::Transform + rule(params: subtree(:p)) do + (p.is_a?(Array) ? p : [p]).each_with_object({}) { |(key, val), h| h[key] = val } + end + + rule(param: { key: simple(:key), value: simple(:val) }) do + [key, val] + end + + rule(quoted_string: simple(:string)) do + string.to_s + end + + rule(token: simple(:string)) do + string.to_s + end + end + def require_signature! render plain: signature_verification_failure_reason, status: signature_verification_failure_code unless signed_request_account end @@ -24,72 +62,40 @@ module SignatureVerification end def signature_key_id - raw_signature = request.headers['Signature'] - signature_params = {} - - raw_signature.split(',').each do |part| - parsed_parts = part.match(/([a-z]+)="([^"]+)"/i) - next if parsed_parts.nil? || parsed_parts.size != 3 - signature_params[parsed_parts[1]] = parsed_parts[2] - end - signature_params['keyId'] + rescue SignatureVerificationError + nil end def signed_request_account return @signed_request_account if defined?(@signed_request_account) - unless signed_request? - @signature_verification_failure_reason = 'Request not signed' - @signed_request_account = nil - return - end + raise SignatureVerificationError, 'Request not signed' unless signed_request? + raise SignatureVerificationError, 'Incompatible request signature. keyId and signature are required' if missing_required_signature_parameters? + raise SignatureVerificationError, 'Unsupported signature algorithm (only rsa-sha256 and hs2019 are supported)' unless %w(rsa-sha256 hs2019).include?(signature_algorithm) + raise SignatureVerificationError, 'Signed request date outside acceptable time window' unless matches_time_window? - if request.headers['Date'].present? && !matches_time_window? - @signature_verification_failure_reason = 'Signed request date outside acceptable time window' - @signed_request_account = nil - return - end - - raw_signature = request.headers['Signature'] - signature_params = {} - - raw_signature.split(',').each do |part| - parsed_parts = part.match(/([a-z]+)="([^"]+)"/i) - next if parsed_parts.nil? || parsed_parts.size != 3 - signature_params[parsed_parts[1]] = parsed_parts[2] - end - - if incompatible_signature?(signature_params) - @signature_verification_failure_reason = 'Incompatible request signature' - @signed_request_account = nil - return - end + verify_signature_strength! account = account_from_key_id(signature_params['keyId']) - if account.nil? - @signature_verification_failure_reason = "Public key not found for key #{signature_params['keyId']}" - @signed_request_account = nil - return - end + raise SignatureVerificationError, "Public key not found for key #{signature_params['keyId']}" if account.nil? signature = Base64.decode64(signature_params['signature']) - compare_signed_string = build_signed_string(signature_params['headers']) + compare_signed_string = build_signed_string return account unless verify_signature(account, signature, compare_signed_string).nil? account = stoplight_wrap_request { account.possibly_stale? ? account.refresh! : account_refresh_key(account) } - if account.nil? - @signature_verification_failure_reason = "Public key not found for key #{signature_params['keyId']}" - @signed_request_account = nil - return - end + raise SignatureVerificationError, "Public key not found for key #{signature_params['keyId']}" if account.nil? return account unless verify_signature(account, signature, compare_signed_string).nil? - @signature_verification_failure_reason = "Verification failed for #{account.username}@#{account.domain} #{account.uri}" + @signature_verification_failure_reason = "Verification failed for #{account.username}@#{account.domain} #{account.uri} using rsa-sha256 (RSASSA-PKCS1-v1_5 with SHA-256)" + @signed_request_account = nil + rescue SignatureVerificationError => e + @signature_verification_failure_reason = e.message @signed_request_account = nil end @@ -99,6 +105,31 @@ module SignatureVerification private + def signature_params + @signature_params ||= begin + raw_signature = request.headers['Signature'] + tree = SignatureParamsParser.new.parse(raw_signature) + SignatureParamsTransformer.new.apply(tree) + end + rescue Parslet::ParseFailed + raise SignatureVerificationError, 'Error parsing signature parameters' + end + + def signature_algorithm + signature_params.fetch('algorithm', 'hs2019') + end + + def signed_headers + signature_params.fetch('headers', signature_algorithm == 'hs2019' ? '(created)' : 'date').downcase.split(' ') + end + + def verify_signature_strength! + raise SignatureVerificationError, 'Mastodon requires the Date header or (created) pseudo-header to be signed' unless signed_headers.include?('date') || signed_headers.include?('(created)') + raise SignatureVerificationError, 'Mastodon requires the Digest header or (request-target) pseudo-header to be signed' unless signed_headers.include?(Request::REQUEST_TARGET) || signed_headers.include?('digest') + raise SignatureVerificationError, 'Mastodon requires the Host header to be signed' unless signed_headers.include?('host') + raise SignatureVerificationError, 'Mastodon requires the Digest header to be signed when doing a POST request' if request.post? && !signed_headers.include?('digest') + end + def verify_signature(account, signature, compare_signed_string) if account.keypair.public_key.verify(OpenSSL::Digest::SHA256.new, signature, compare_signed_string) @signed_request_account = account @@ -108,12 +139,20 @@ module SignatureVerification nil end - def build_signed_string(signed_headers) - signed_headers = 'date' if signed_headers.blank? - - signed_headers.downcase.split(' ').map do |signed_header| + def build_signed_string + signed_headers.map do |signed_header| if signed_header == Request::REQUEST_TARGET "#{Request::REQUEST_TARGET}: #{request.method.downcase} #{request.path}" + elsif signed_header == '(created)' + raise SignatureVerificationError, 'Invalid pseudo-header (created) for rsa-sha256' unless signature_algorithm == 'hs2019' + raise SignatureVerificationError, 'Pseudo-header (created) used but corresponding argument missing' if signature_params['created'].blank? + + "(created): #{signature_params['created']}" + elsif signed_header == '(expires)' + raise SignatureVerificationError, 'Invalid pseudo-header (expires) for rsa-sha256' unless signature_algorithm == 'hs2019' + raise SignatureVerificationError, 'Pseudo-header (expires) used but corresponding argument missing' if signature_params['expires'].blank? + + "(expires): #{signature_params['expires']}" elsif signed_header == 'digest' "digest: #{body_digest}" else @@ -123,13 +162,28 @@ module SignatureVerification end def matches_time_window? + created_time = nil + expires_time = nil + begin - time_sent = Time.httpdate(request.headers['Date']) + if signature_algorithm == 'hs2019' && signature_params['created'].present? + created_time = Time.at(signature_params['created'].to_i).utc + elsif request.headers['Date'].present? + created_time = Time.httpdate(request.headers['Date']).utc + end + + expires_time = Time.at(signature_params['expires'].to_i).utc if signature_params['expires'].present? rescue ArgumentError return false end - (Time.now.utc - time_sent).abs <= 12.hours + expires_time ||= created_time + 5.minutes unless created_time.nil? + expires_time = [expires_time, created_time + EXPIRATION_WINDOW_LIMIT].min unless created_time.nil? + + return false if created_time.present? && created_time > Time.now.utc + CLOCK_SKEW_MARGIN + return false if expires_time.present? && Time.now.utc > expires_time + CLOCK_SKEW_MARGIN + + true end def body_digest @@ -140,9 +194,8 @@ module SignatureVerification name.split(/-/).map(&:capitalize).join('-') end - def incompatible_signature?(signature_params) - signature_params['keyId'].blank? || - signature_params['signature'].blank? + def missing_required_signature_parameters? + signature_params['keyId'].blank? || signature_params['signature'].blank? end def account_from_key_id(key_id) From a583e540232fe7f3c0902dec0ba97252eb4357cc Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 19 Oct 2020 15:58:53 +0200 Subject: [PATCH 20/23] Bump version to 3.2.1 --- CHANGELOG.md | 29 +++++++++++++++++++++++++++++ lib/mastodon/version.rb | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18790e860..6f94ebea2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,35 @@ Changelog All notable changes to this project will be documented in this file. +## [3.2.1] - 2020-10-19 +### Added + +- Add support for latest HTTP Signatures spec draft ([ThibG](https://github.com/tootsuite/mastodon/pull/14556)) +- Add support for inlined objects in ActivityPub `to`/`cc` ([ThibG](https://github.com/tootsuite/mastodon/pull/14514)) + +### Changed + +- Change actors to not be served at all without authentication in limited federation mode ([ThibG](https://github.com/tootsuite/mastodon/pull/14800)) + - Previously, a bare version of an actor was served when not authenticated, i.e. username and public key + - Because all actor fetch requests are signed using a separate system actor, that is no longer required + +### Fixed + +- Fix `tootctl media` commands not recognizing very large IDs ([ThibG](https://github.com/tootsuite/mastodon/pull/14536)) +- Fix crash when failing to load emoji picker in web UI ([ThibG](https://github.com/tootsuite/mastodon/pull/14525)) +- Fix contrast requirements in thumbnail color extraction ([ThibG](https://github.com/tootsuite/mastodon/pull/14464)) +- Fix audio/video player not using `CDN_HOST` on public pages ([ThibG](https://github.com/tootsuite/mastodon/pull/14486)) +- Fix private boost icon not being used on public pages ([OmmyZhang](https://github.com/tootsuite/mastodon/pull/14471)) +- Fix audio player on Safari in web UI ([ThibG](https://github.com/tootsuite/mastodon/pull/14485), [ThibG](https://github.com/tootsuite/mastodon/pull/14465)) +- Fix dereferencing remote statuses not using the correct account for signature when receiving a targeted inbox delivery ([ThibG](https://github.com/tootsuite/mastodon/pull/14656)) +- Fix nil error in `tootctl media remove` ([noellabo](https://github.com/tootsuite/mastodon/pull/14657)) +- Fix videos with near-60 fps being rejected ([Gargron](https://github.com/tootsuite/mastodon/pull/14684)) +- Fix reported statuses not being included in warning e-mail ([Gargron](https://github.com/tootsuite/mastodon/pull/14778)) +- Fix `Reject` activities of `Follow` objects not correctly destroying a follow relationship ([ThibG](https://github.com/tootsuite/mastodon/pull/14479)) +- Fix inefficiencies in fan-out-on-write service ([Gargron](https://github.com/tootsuite/mastodon/pull/14682), [noellabo](https://github.com/tootsuite/mastodon/pull/14709)) +- Fix timeout errors when trying to webfinger some IPv6 configurations ([Gargron](https://github.com/tootsuite/mastodon/pull/14919)) +- Fix files served as `application/octet-stream` being rejected without attempting mime type detection ([ThibG](https://github.com/tootsuite/mastodon/pull/14452)) + ## [3.2.0] - 2020-07-27 ### Added diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 7aa6cb2c7..344ea996e 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 0 + 1 end def flags From 406adfca275909111153dfde91626a849fed5a1f Mon Sep 17 00:00:00 2001 From: ThibG Date: Fri, 18 Dec 2020 23:31:14 +0100 Subject: [PATCH 21/23] Backport fixes to 3.2 (#15360) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix 2FA/sign-in token sessions being valid after password change (#14802) If someone tries logging in to an account and is prompted for a 2FA code or sign-in token, even if the account's password or e-mail is updated in the meantime, the session will show the prompt and allow the login process to complete with a valid 2FA code or sign-in token * Fix Move handler not being triggered when failing to fetch target (#15107) When failing to fetch the target account, the ProcessingWorker fails as expected, but since it hasn't cleared the `move_in_progress` flag, the next attempt at processing skips the `Move` activity altogether. This commit changes it to clear the flag when encountering any unexpected error on fetching the target account. This is likely to occur because, of, e.g., a timeout, when many instances query the same actor at the same time. * Fix slow distinct queries where grouped queries are faster (#15287) About 2x speed-up on inboxes query * Fix possible inconsistencies in tag search (#14906) Do not downcase the queried tag before passing it to postgres when searching: - tags are not downcased on creation - `arel_table[:name].lower.matches(pattern)` generates an ILIKE anyway - if Postgres and Rails happen to use different case-folding rules, downcasing before query but not before insertion may mean that some tags with some casings are not searchable * Fix updating account counters when account_stat is not yet created (#15108) * Fix account processing failing because of large collections (#15027) Fixes #15025 * Fix downloading remote media files when server returns empty filename (#14867) Fixes #14817 * Fix webfinger redirect handling in ResolveAccountService (#15187) * Fix webfinger redirect handling in ResolveAccountService ResolveAccountService#process_webfinger! handled a one-step webfinger redirection, but only accepting the result if it matched the exact URI passed as input, defeating the point of a redirection check. Instead, use the same logic as in `ActivityPub::FetchRemoteAccountService`, updating the resulting `acct:` URI with the result of the first webfinger query. * Add tests * Remove dependency on unused and unmaintained http_parser.rb gem (#14574) It seems that years ago, the “http” gem dependend on the “http_parser.rb” gem (it now depends on the “http-parser” gem), and, still years ago, we pulled it from git in order to benefit from a bugfix that wasn't released yet (#7467). * Add tootctl maintenance fix-duplicates (#14860, #15201, #15264, #15349, #15359) * Fix old migration script not being able to run if it fails midway (#15361) * Fix old migration script not being able to run if it fails midway Improve the robustness of a migration script likely to fail because of database corruption so it can run again once database corruptions are fixed. * Display a specific error message in case of index corruption Co-authored-by: Eugen Rochko Co-authored-by: Claire Co-authored-by: Eugen Rochko Co-authored-by: Claire --- Gemfile | 1 - Gemfile.lock | 9 - app/controllers/accounts_controller.rb | 2 +- app/controllers/admin/statuses_controller.rb | 2 +- app/controllers/api/base_controller.rb | 2 +- app/controllers/auth/sessions_controller.rb | 22 +- .../sign_in_token_authentication_concern.rb | 16 +- .../two_factor_authentication_concern.rb | 16 +- .../concerns/user_tracking_concern.rb | 7 +- app/lib/activitypub/activity/move.rb | 3 + app/models/account.rb | 2 +- app/models/account_stat.rb | 25 +- app/models/form/account_batch.rb | 2 +- app/models/tag.rb | 2 +- app/models/user.rb | 25 +- .../activitypub/process_account_service.rb | 2 +- app/services/resolve_account_service.rb | 23 +- ...3_add_fixed_lowercase_index_to_accounts.rb | 26 +- lib/cli.rb | 4 + lib/mastodon/maintenance_cli.rb | 618 ++++++++++++++++++ lib/paperclip/response_with_limit_adapter.rb | 2 +- .../auth/sessions_controller_spec.rb | 12 +- spec/models/account_spec.rb | 23 + spec/services/resolve_account_service_spec.rb | 52 +- 24 files changed, 821 insertions(+), 77 deletions(-) create mode 100644 lib/mastodon/maintenance_cli.rb diff --git a/Gemfile b/Gemfile index 0891371be..5ac1c79c1 100644 --- a/Gemfile +++ b/Gemfile @@ -60,7 +60,6 @@ gem 'health_check', git: 'https://github.com/ianheggie/health_check', ref: '0b79 gem 'htmlentities', '~> 4.3' gem 'http', '~> 4.4' gem 'http_accept_language', '~> 2.1' -gem 'http_parser.rb', '~> 0.6', git: 'https://github.com/tmm1/http_parser.rb', ref: '54b17ba8c7d8d20a16dfc65d1775241833219cf2', submodules: true gem 'httplog', '~> 1.4.3' gem 'idn-ruby', require: 'idn' gem 'kaminari', '~> 1.2' diff --git a/Gemfile.lock b/Gemfile.lock index 62c20bf07..c3216870d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -13,14 +13,6 @@ GIT specs: posix-spawn (0.3.13) -GIT - remote: https://github.com/tmm1/http_parser.rb - revision: 54b17ba8c7d8d20a16dfc65d1775241833219cf2 - ref: 54b17ba8c7d8d20a16dfc65d1775241833219cf2 - submodules: true - specs: - http_parser.rb (0.6.1) - GIT remote: https://github.com/witgo/nilsimsa revision: fd184883048b922b176939f851338d0a4971a532 @@ -709,7 +701,6 @@ DEPENDENCIES htmlentities (~> 4.3) http (~> 4.4) http_accept_language (~> 2.1) - http_parser.rb (~> 0.6)! httplog (~> 1.4.3) i18n-tasks (~> 0.9) idn-ruby diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index f5e82692e..b42f711f0 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -82,7 +82,7 @@ class AccountsController < ApplicationController end def account_media_status_ids - @account.media_attachments.attached.reorder(nil).select(:status_id).distinct + @account.media_attachments.attached.reorder(nil).select(:status_id).group(:status_id) end def no_replies_scope diff --git a/app/controllers/admin/statuses_controller.rb b/app/controllers/admin/statuses_controller.rb index 650195034..d7c192f0d 100644 --- a/app/controllers/admin/statuses_controller.rb +++ b/app/controllers/admin/statuses_controller.rb @@ -14,7 +14,7 @@ module Admin @statuses = @account.statuses.where(visibility: [:public, :unlisted]) if params[:media] - account_media_status_ids = @account.media_attachments.attached.reorder(nil).select(:status_id).distinct + account_media_status_ids = @account.media_attachments.attached.reorder(nil).select(:status_id).group(:status_id) @statuses.merge!(Status.where(id: account_media_status_ids)) end diff --git a/app/controllers/api/base_controller.rb b/app/controllers/api/base_controller.rb index 045e7dd26..055cd416d 100644 --- a/app/controllers/api/base_controller.rb +++ b/app/controllers/api/base_controller.rb @@ -102,7 +102,7 @@ class Api::BaseController < ApplicationController elsif !current_user.approved? render json: { error: 'Your login is currently pending approval' }, status: 403 else - set_user_activity + update_user_sign_in end end diff --git a/app/controllers/auth/sessions_controller.rb b/app/controllers/auth/sessions_controller.rb index 1fd755334..783338450 100644 --- a/app/controllers/auth/sessions_controller.rb +++ b/app/controllers/auth/sessions_controller.rb @@ -7,6 +7,7 @@ class Auth::SessionsController < Devise::SessionsController skip_before_action :require_no_authentication, only: [:create] skip_before_action :require_functional! + skip_before_action :update_user_sign_in include TwoFactorAuthenticationConcern include SignInTokenAuthenticationConcern @@ -24,6 +25,7 @@ class Auth::SessionsController < Devise::SessionsController def create super do |resource| + resource.update_sign_in!(request, new_sign_in: true) remember_me(resource) flash.delete(:notice) end @@ -41,7 +43,7 @@ class Auth::SessionsController < Devise::SessionsController def find_user if session[:attempt_user_id] - User.find(session[:attempt_user_id]) + User.find_by(id: session[:attempt_user_id]) else user = User.authenticate_with_ldap(user_params) if Devise.ldap_authentication user ||= User.authenticate_with_pam(user_params) if Devise.pam_authentication @@ -74,6 +76,7 @@ class Auth::SessionsController < Devise::SessionsController def require_no_authentication super + # Delete flash message that isn't entirely useful and may be confusing in # most cases because /web doesn't display/clear flash messages. flash.delete(:alert) if flash[:alert] == I18n.t('devise.failure.already_authenticated') @@ -91,13 +94,30 @@ class Auth::SessionsController < Devise::SessionsController def home_paths(resource) paths = [about_path] + if single_user_mode? && resource.is_a?(User) paths << short_account_path(username: resource.account) end + paths end def continue_after? truthy_param?(:continue) end + + def restart_session + clear_attempt_from_session + redirect_to new_user_session_path, alert: I18n.t('devise.failure.timeout') + end + + def set_attempt_session(user) + session[:attempt_user_id] = user.id + session[:attempt_user_updated_at] = user.updated_at.to_s + end + + def clear_attempt_from_session + session.delete(:attempt_user_id) + session.delete(:attempt_user_updated_at) + end end diff --git a/app/controllers/concerns/sign_in_token_authentication_concern.rb b/app/controllers/concerns/sign_in_token_authentication_concern.rb index 91f813acc..3c95a4afd 100644 --- a/app/controllers/concerns/sign_in_token_authentication_concern.rb +++ b/app/controllers/concerns/sign_in_token_authentication_concern.rb @@ -18,7 +18,9 @@ module SignInTokenAuthenticationConcern def authenticate_with_sign_in_token user = self.resource = find_user - if user_params[:sign_in_token_attempt].present? && session[:attempt_user_id] + if user.present? && session[:attempt_user_id].present? && session[:attempt_user_updated_at] != user.updated_at.to_s + restart_session + elsif user_params.key?(:sign_in_token_attempt) && session[:attempt_user_id] authenticate_with_sign_in_token_attempt(user) elsif user.present? && user.external_or_valid_password?(user_params[:password]) prompt_for_sign_in_token(user) @@ -27,7 +29,7 @@ module SignInTokenAuthenticationConcern def authenticate_with_sign_in_token_attempt(user) if valid_sign_in_token_attempt?(user) - session.delete(:attempt_user_id) + clear_attempt_from_session remember_me(user) sign_in(user) else @@ -42,10 +44,10 @@ module SignInTokenAuthenticationConcern UserMailer.sign_in_token(user, request.remote_ip, request.user_agent, Time.now.utc.to_s).deliver_later! end - set_locale do - session[:attempt_user_id] = user.id - @body_classes = 'lighter' - render :sign_in_token - end + set_attempt_session(user) + + @body_classes = 'lighter' + + set_locale { render :sign_in_token } end end diff --git a/app/controllers/concerns/two_factor_authentication_concern.rb b/app/controllers/concerns/two_factor_authentication_concern.rb index daafe56f4..f54a3c829 100644 --- a/app/controllers/concerns/two_factor_authentication_concern.rb +++ b/app/controllers/concerns/two_factor_authentication_concern.rb @@ -21,7 +21,9 @@ module TwoFactorAuthenticationConcern def authenticate_with_two_factor user = self.resource = find_user - if user_params[:otp_attempt].present? && session[:attempt_user_id] + if user.present? && session[:attempt_user_id].present? && session[:attempt_user_updated_at] != user.updated_at.to_s + restart_session + elsif user_params[:otp_attempt].present? && session[:attempt_user_id] authenticate_with_two_factor_attempt(user) elsif user.present? && user.external_or_valid_password?(user_params[:password]) prompt_for_two_factor(user) @@ -30,7 +32,7 @@ module TwoFactorAuthenticationConcern def authenticate_with_two_factor_attempt(user) if valid_otp_attempt?(user) - session.delete(:attempt_user_id) + clear_attempt_from_session remember_me(user) sign_in(user) else @@ -40,10 +42,10 @@ module TwoFactorAuthenticationConcern end def prompt_for_two_factor(user) - set_locale do - session[:attempt_user_id] = user.id - @body_classes = 'lighter' - render :two_factor - end + set_attempt_session(user) + + @body_classes = 'lighter' + + set_locale { render :two_factor } end end diff --git a/app/controllers/concerns/user_tracking_concern.rb b/app/controllers/concerns/user_tracking_concern.rb index be10705fc..efda37fae 100644 --- a/app/controllers/concerns/user_tracking_concern.rb +++ b/app/controllers/concerns/user_tracking_concern.rb @@ -6,14 +6,13 @@ module UserTrackingConcern UPDATE_SIGN_IN_HOURS = 24 included do - before_action :set_user_activity + before_action :update_user_sign_in end private - def set_user_activity - return unless user_needs_sign_in_update? - current_user.update_tracked_fields!(request) + def update_user_sign_in + current_user.update_sign_in!(request) if user_needs_sign_in_update? end def user_needs_sign_in_update? diff --git a/app/lib/activitypub/activity/move.rb b/app/lib/activitypub/activity/move.rb index 2103f503f..7e073f64d 100644 --- a/app/lib/activitypub/activity/move.rb +++ b/app/lib/activitypub/activity/move.rb @@ -20,6 +20,9 @@ class ActivityPub::Activity::Move < ActivityPub::Activity # Initiate a re-follow for each follower MoveWorker.perform_async(origin_account.id, target_account.id) + rescue + unmark_as_processing! + raise end private diff --git a/app/models/account.rb b/app/models/account.rb index 6b7ebda9e..9288c7fe8 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -415,7 +415,7 @@ class Account < ApplicationRecord end def inboxes - urls = reorder(nil).where(protocol: :activitypub).pluck(Arel.sql("distinct coalesce(nullif(accounts.shared_inbox_url, ''), accounts.inbox_url)")) + urls = reorder(nil).where(protocol: :activitypub).group(:preferred_inbox_url).pluck(Arel.sql("coalesce(nullif(accounts.shared_inbox_url, ''), accounts.inbox_url) AS preferred_inbox_url")) DeliveryFailureTracker.without_unavailable(urls) end diff --git a/app/models/account_stat.rb b/app/models/account_stat.rb index c84e4217c..e70b54d79 100644 --- a/app/models/account_stat.rb +++ b/app/models/account_stat.rb @@ -21,26 +21,26 @@ class AccountStat < ApplicationRecord def increment_count!(key) update(attributes_for_increment(key)) - rescue ActiveRecord::StaleObjectError + rescue ActiveRecord::StaleObjectError, ActiveRecord::RecordNotUnique begin reload_with_id rescue ActiveRecord::RecordNotFound - # Nothing to do - else - retry + return end + + retry end def decrement_count!(key) - update(key => [public_send(key) - 1, 0].max) - rescue ActiveRecord::StaleObjectError + update(attributes_for_decrement(key)) + rescue ActiveRecord::StaleObjectError, ActiveRecord::RecordNotUnique begin reload_with_id rescue ActiveRecord::RecordNotFound - # Nothing to do - else - retry + return end + + retry end private @@ -51,8 +51,13 @@ class AccountStat < ApplicationRecord attrs end + def attributes_for_decrement(key) + attrs = { key => [public_send(key) - 1, 0].max } + attrs + end + def reload_with_id - self.id = find_by!(account: account).id if new_record? + self.id = self.class.find_by!(account: account).id if new_record? reload end end diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb index 0b285fde9..fc6ab5c90 100644 --- a/app/models/form/account_batch.rb +++ b/app/models/form/account_batch.rb @@ -43,7 +43,7 @@ class Form::AccountBatch end def account_domains - accounts.pluck(Arel.sql('distinct domain')).compact + accounts.group(:domain).pluck(:domain).compact end def accounts diff --git a/app/models/tag.rb b/app/models/tag.rb index bce76fc16..0870afc41 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -126,7 +126,7 @@ class Tag < ApplicationRecord end def search_for(term, limit = 5, offset = 0, options = {}) - normalized_term = normalize(term.strip).mb_chars.downcase.to_s + normalized_term = normalize(term.strip) pattern = sanitize_sql_like(normalized_term) + '%' query = Tag.listable.where(arel_table[:name].lower.matches(pattern)) query = query.where(arel_table[:name].lower.eq(normalized_term).or(arel_table[:reviewed_at].not_eq(nil))) if options[:exclude_unreviewed] diff --git a/app/models/user.rb b/app/models/user.rb index 306e2d435..0efeed9f7 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -61,7 +61,7 @@ class User < ApplicationRecord devise :two_factor_backupable, otp_number_of_backup_codes: 10 - devise :registerable, :recoverable, :rememberable, :trackable, :validatable, + devise :registerable, :recoverable, :rememberable, :validatable, :confirmable include Omniauthable @@ -161,6 +161,24 @@ class User < ApplicationRecord prepare_new_user! if new_user && approved? end + def update_sign_in!(request, new_sign_in: false) + old_current, new_current = current_sign_in_at, Time.now.utc + self.last_sign_in_at = old_current || new_current + self.current_sign_in_at = new_current + + old_current, new_current = current_sign_in_ip, request.remote_ip + self.last_sign_in_ip = old_current || new_current + self.current_sign_in_ip = new_current + + if new_sign_in + self.sign_in_count ||= 0 + self.sign_in_count += 1 + end + + save(validate: false) unless new_record? + prepare_returning_user! + end + def pending? !approved? end @@ -192,11 +210,6 @@ class User < ApplicationRecord prepare_new_user! end - def update_tracked_fields!(request) - super - prepare_returning_user! - end - def disable_two_factor! self.otp_required_for_login = false otp_backup_codes&.clear diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index 85b915ec6..9f95f1950 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -196,7 +196,7 @@ class ActivityPub::ProcessAccountService < BaseService total_items = collection.is_a?(Hash) && collection['totalItems'].present? && collection['totalItems'].is_a?(Numeric) ? collection['totalItems'] : nil has_first_page = collection.is_a?(Hash) && collection['first'].present? @collections[type] = [total_items, has_first_page] - rescue HTTP::Error, OpenSSL::SSL::SSLError + rescue HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::LengthValidationError @collections[type] = [nil, nil] end diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb index 3f7bb7cc5..eee1de51a 100644 --- a/app/services/resolve_account_service.rb +++ b/app/services/resolve_account_service.rb @@ -31,6 +31,7 @@ class ResolveAccountService < BaseService # At this point we are in need of a Webfinger query, which may # yield us a different username/domain through a redirect process_webfinger!(@uri) + @domain = nil if TagManager.instance.local_domain?(@domain) # Because the username/domain pair may be different than what # we already checked, we need to check if we've already got @@ -75,21 +76,27 @@ class ResolveAccountService < BaseService @uri = [@username, @domain].compact.join('@') end - def process_webfinger!(uri, redirected = false) + def process_webfinger!(uri) @webfinger = webfinger!("acct:#{uri}") - confirmed_username, confirmed_domain = @webfinger.subject.gsub(/\Aacct:/, '').split('@') + confirmed_username, confirmed_domain = split_acct(@webfinger.subject) if confirmed_username.casecmp(@username).zero? && confirmed_domain.casecmp(@domain).zero? @username = confirmed_username @domain = confirmed_domain - @uri = uri - elsif !redirected - return process_webfinger!("#{confirmed_username}@#{confirmed_domain}", true) - else - raise WebfingerRedirectError, "The URI #{uri} tries to hijack #{@username}@#{@domain}" + return end - @domain = nil if TagManager.instance.local_domain?(@domain) + # Account doesn't match, so it may have been redirected + @webfinger = webfinger!("acct:#{confirmed_username}@#{confirmed_domain}") + @username, @domain = split_acct(@webfinger.subject) + + unless confirmed_username.casecmp(@username).zero? && confirmed_domain.casecmp(@domain).zero? + raise WebfingerRedirectError, "The URI #{uri} tries to hijack #{@username}@#{@domain}" + end + end + + def split_acct(acct) + acct.gsub(/\Aacct:/, '').split('@') end def process_account! diff --git a/db/migrate/20200620164023_add_fixed_lowercase_index_to_accounts.rb b/db/migrate/20200620164023_add_fixed_lowercase_index_to_accounts.rb index c5688681f..c3aa8e33c 100644 --- a/db/migrate/20200620164023_add_fixed_lowercase_index_to_accounts.rb +++ b/db/migrate/20200620164023_add_fixed_lowercase_index_to_accounts.rb @@ -1,10 +1,30 @@ class AddFixedLowercaseIndexToAccounts < ActiveRecord::Migration[5.2] disable_ddl_transaction! + class CorruptionError < StandardError + def cause + nil + end + + def backtrace + [] + end + end + def up - rename_index :accounts, 'index_accounts_on_username_and_domain_lower', 'old_index_accounts_on_username_and_domain_lower' unless index_name_exists?(:accounts, 'old_index_accounts_on_username_and_domain_lower') - add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true, algorithm: :concurrently - remove_index :accounts, name: 'old_index_accounts_on_username_and_domain_lower' + if index_name_exists?(:accounts, 'old_index_accounts_on_username_and_domain_lower') && index_name_exists?(:accounts, 'index_accounts_on_username_and_domain_lower') + remove_index :accounts, name: 'index_accounts_on_username_and_domain_lower' + elsif index_name_exists?(:accounts, 'index_accounts_on_username_and_domain_lower') + rename_index :accounts, 'index_accounts_on_username_and_domain_lower', 'old_index_accounts_on_username_and_domain_lower' + end + + begin + add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true, algorithm: :concurrently + rescue ActiveRecord::RecordNotUnique + raise CorruptionError, 'Migration failed because of index corruption, see https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/#fixing' + end + + remove_index :accounts, name: 'old_index_accounts_on_username_and_domain_lower' if index_name_exists?(:accounts, 'old_index_accounts_on_username_and_domain_lower') end def down diff --git a/lib/cli.rb b/lib/cli.rb index 9162144cc..38df8abcc 100644 --- a/lib/cli.rb +++ b/lib/cli.rb @@ -13,6 +13,7 @@ require_relative 'mastodon/preview_cards_cli' require_relative 'mastodon/cache_cli' require_relative 'mastodon/upgrade_cli' require_relative 'mastodon/email_domain_blocks_cli' +require_relative 'mastodon/maintenance_cli' require_relative 'mastodon/version' module Mastodon @@ -57,6 +58,9 @@ module Mastodon desc 'email_domain_blocks SUBCOMMAND ...ARGS', 'Manage e-mail domain blocks' subcommand 'email_domain_blocks', Mastodon::EmailDomainBlocksCLI + desc 'maintenance SUBCOMMAND ...ARGS', 'Various maintenance utilities' + subcommand 'maintenance', Mastodon::MaintenanceCLI + option :dry_run, type: :boolean desc 'self-destruct', 'Erase the server from the federation' long_desc <<~LONG_DESC diff --git a/lib/mastodon/maintenance_cli.rb b/lib/mastodon/maintenance_cli.rb new file mode 100644 index 000000000..822051ceb --- /dev/null +++ b/lib/mastodon/maintenance_cli.rb @@ -0,0 +1,618 @@ +# frozen_string_literal: true + +require 'tty-prompt' +require_relative '../../config/boot' +require_relative '../../config/environment' +require_relative 'cli_helper' + +module Mastodon + class MaintenanceCLI < Thor + include CLIHelper + + def self.exit_on_failure? + true + end + + MIN_SUPPORTED_VERSION = 2019_10_01_213028 + MAX_SUPPORTED_VERSION = 2020_12_18_054746 + + # Stubs to enjoy ActiveRecord queries while not depending on a particular + # version of the code/database + + class Status < ApplicationRecord; end + class StatusPin < ApplicationRecord; end + class Poll < ApplicationRecord; end + class Report < ApplicationRecord; end + class Tombstone < ApplicationRecord; end + class Favourite < ApplicationRecord; end + class Follow < ApplicationRecord; end + class FollowRequest < ApplicationRecord; end + class Block < ApplicationRecord; end + class Mute < ApplicationRecord; end + class AccountIdentityProof < ApplicationRecord; end + class AccountModerationNote < ApplicationRecord; end + class AccountPin < ApplicationRecord; end + class ListAccount < ApplicationRecord; end + class PollVote < ApplicationRecord; end + class Mention < ApplicationRecord; end + class AccountDomainBlock < ApplicationRecord; end + class AnnouncementReaction < ApplicationRecord; end + class FeaturedTag < ApplicationRecord; end + class CustomEmoji < ApplicationRecord; end + class CustomEmojiCategory < ApplicationRecord; end + class Bookmark < ApplicationRecord; end + class WebauthnCredential < ApplicationRecord; end + + class PreviewCard < ApplicationRecord + self.inheritance_column = false + end + + class MediaAttachment < ApplicationRecord + self.inheritance_column = nil + end + + class AccountStat < ApplicationRecord + belongs_to :account, inverse_of: :account_stat + end + + # Dummy class, to make migration possible across version changes + class Account < ApplicationRecord + has_one :user, inverse_of: :account + has_one :account_stat, inverse_of: :account + + scope :local, -> { where(domain: nil) } + + def local? + domain.nil? + end + + def acct + local? ? username : "#{username}@#{domain}" + end + + # This is a duplicate of the AccountMerging concern because we need it to + # be independent from code version. + def merge_with!(other_account) + # Since it's the same remote resource, the remote resource likely + # already believes we are following/blocking, so it's safe to + # re-attribute the relationships too. However, during the presence + # of the index bug users could have *also* followed the reference + # account already, therefore mass update will not work and we need + # to check for (and skip past) uniqueness errors + + owned_classes = [ + Status, StatusPin, MediaAttachment, Poll, Report, Tombstone, Favourite, + Follow, FollowRequest, Block, Mute, AccountIdentityProof, + AccountModerationNote, AccountPin, AccountStat, ListAccount, + PollVote, Mention + ] + owned_classes << AccountDeletionRequest if ActiveRecord::Base.connection.table_exists?(:account_deletion_requests) + owned_classes << AccountNote if ActiveRecord::Base.connection.table_exists?(:account_notes) + + owned_classes.each do |klass| + klass.where(account_id: other_account.id).find_each do |record| + begin + record.update_attribute(:account_id, id) + rescue ActiveRecord::RecordNotUnique + next + end + end + end + + target_classes = [Follow, FollowRequest, Block, Mute, AccountModerationNote, AccountPin] + target_classes << AccountNote if ActiveRecord::Base.connection.table_exists?(:account_notes) + + target_classes.each do |klass| + klass.where(target_account_id: other_account.id).find_each do |record| + begin + record.update_attribute(:target_account_id, id) + rescue ActiveRecord::RecordNotUnique + next + end + end + end + end + end + + class User < ApplicationRecord + belongs_to :account, inverse_of: :user + end + + desc 'fix-duplicates', 'Fix duplicates in database and rebuild indexes' + long_desc <<~LONG_DESC + Delete or merge duplicate accounts, statuses, emojis, etc. and rebuild indexes. + + This is useful if your database indexes are corrupted because of issues such as https://wiki.postgresql.org/wiki/Locale_data_changes + + Mastodon has to be stopped to run this task, which will take a long time and may be destructive. + LONG_DESC + def fix_duplicates + @prompt = TTY::Prompt.new + + if ActiveRecord::Migrator.current_version < MIN_SUPPORTED_VERSION + @prompt.warn 'Your version of the database schema is too old and is not supported by this script.' + @prompt.warn 'Please update to at least Mastodon 3.0.0 before running this script.' + exit(1) + elsif ActiveRecord::Migrator.current_version > MAX_SUPPORTED_VERSION + @prompt.warn 'Your version of the database schema is more recent than this script, this may cause unexpected errors.' + exit(1) unless @prompt.yes?('Continue anyway?') + end + + @prompt.warn 'This task will take a long time to run and is potentially destructive.' + @prompt.warn 'Please make sure to stop Mastodon and have a backup.' + exit(1) unless @prompt.yes?('Continue?') + + deduplicate_accounts! + deduplicate_users! + deduplicate_account_domain_blocks! + deduplicate_account_identity_proofs! + deduplicate_announcement_reactions! + deduplicate_conversations! + deduplicate_custom_emojis! + deduplicate_custom_emoji_categories! + deduplicate_domain_allows! + deduplicate_domain_blocks! + deduplicate_unavailable_domains! + deduplicate_email_domain_blocks! + deduplicate_media_attachments! + deduplicate_preview_cards! + deduplicate_statuses! + deduplicate_tags! + deduplicate_webauthn_credentials! + + Rails.cache.clear + + @prompt.say 'Finished!' + end + + private + + def deduplicate_accounts! + remove_index_if_exists!(:accounts, 'index_accounts_on_username_and_domain_lower') + + @prompt.say 'Deduplicating accounts… for local accounts, you will be asked to chose which account to keep unchanged.' + + find_duplicate_accounts.each do |row| + accounts = Account.where(id: row['ids'].split(',')).to_a + + if accounts.first.local? + deduplicate_local_accounts!(accounts) + else + deduplicate_remote_accounts!(accounts) + end + end + + @prompt.say 'Restoring index_accounts_on_username_and_domain_lower…' + if ActiveRecord::Migrator.current_version < 20200620164023 + ActiveRecord::Base.connection.add_index :accounts, 'lower (username), lower(domain)', name: 'index_accounts_on_username_and_domain_lower', unique: true + else + ActiveRecord::Base.connection.add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true + end + end + + def deduplicate_users! + remove_index_if_exists!(:users, 'index_users_on_confirmation_token') + remove_index_if_exists!(:users, 'index_users_on_email') + remove_index_if_exists!(:users, 'index_users_on_remember_token') + remove_index_if_exists!(:users, 'index_users_on_reset_password_token') + + @prompt.say 'Deduplicating user records…' + + # Deduplicating email + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users GROUP BY email HAVING count(*) > 1").each do |row| + users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse + ref_user = users.shift + @prompt.warn "Multiple users registered with e-mail address #{ref_user.email}." + @prompt.warn "e-mail will be disabled for the following accounts: #{user.map(&:account).map(&:acct).join(', ')}" + @prompt.warn 'Please reach out to them and set another address with `tootctl account modify` or delete them.' + + i = 0 + users.each do |user| + user.update!(email: "#{i} " + user.email) + end + end + + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE confirmation_token IS NOT NULL GROUP BY confirmation_token HAVING count(*) > 1").each do |row| + users = User.where(id: row['ids'].split(',')).sort_by(&:created_at).reverse.drop(1) + @prompt.warn "Unsetting confirmation token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}" + + users.each do |user| + user.update!(confirmation_token: nil) + end + end + + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1").each do |row| + users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1) + @prompt.warn "Unsetting remember token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}" + + users.each do |user| + user.update!(remember_token: nil) + end + end + + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE reset_password_token IS NOT NULL GROUP BY reset_password_token HAVING count(*) > 1").each do |row| + users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1) + @prompt.warn "Unsetting password reset token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}" + + users.each do |user| + user.update!(reset_password_token: nil) + end + end + + @prompt.say 'Restoring users indexes…' + ActiveRecord::Base.connection.add_index :users, ['confirmation_token'], name: 'index_users_on_confirmation_token', unique: true + ActiveRecord::Base.connection.add_index :users, ['email'], name: 'index_users_on_email', unique: true + ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true + ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true + end + + def deduplicate_account_domain_blocks! + remove_index_if_exists!(:account_domain_blocks, 'index_account_domain_blocks_on_account_id_and_domain') + + @prompt.say 'Removing duplicate account domain blocks…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_domain_blocks GROUP BY account_id, domain HAVING count(*) > 1").each do |row| + AccountDomainBlock.where(id: row['ids'].split(',').drop(1)).delete_all + end + + @prompt.say 'Restoring account domain blocks indexes…' + ActiveRecord::Base.connection.add_index :account_domain_blocks, ['account_id', 'domain'], name: 'index_account_domain_blocks_on_account_id_and_domain', unique: true + end + + def deduplicate_account_identity_proofs! + remove_index_if_exists!(:account_identity_proofs, 'index_account_proofs_on_account_and_provider_and_username') + + @prompt.say 'Removing duplicate account identity proofs…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_identity_proofs GROUP BY account_id, provider, provider_username HAVING count(*) > 1").each do |row| + AccountIdentityProof.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring account identity proofs indexes…' + ActiveRecord::Base.connection.add_index :account_identity_proofs, ['account_id', 'provider', 'provider_username'], name: 'index_account_proofs_on_account_and_provider_and_username', unique: true + end + + def deduplicate_announcement_reactions! + return unless ActiveRecord::Base.connection.table_exists?(:announcement_reactions) + + remove_index_if_exists!(:announcement_reactions, 'index_announcement_reactions_on_account_id_and_announcement_id') + + @prompt.say 'Removing duplicate account identity proofs…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM announcement_reactions GROUP BY account_id, announcement_id, name HAVING count(*) > 1").each do |row| + AnnouncementReaction.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring announcement_reactions indexes…' + ActiveRecord::Base.connection.add_index :announcement_reactions, ['account_id', 'announcement_id', 'name'], name: 'index_announcement_reactions_on_account_id_and_announcement_id', unique: true + end + + def deduplicate_conversations! + remove_index_if_exists!(:conversations, 'index_conversations_on_uri') + + @prompt.say 'Deduplicating conversations…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM conversations WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row| + conversations = Conversation.where(id: row['ids'].split(',')).sort_by(&:id).reverse + + ref_conversation = conversations.shift + + conversations.each do |other| + merge_conversations!(ref_conversation, other) + other.destroy + end + end + + @prompt.say 'Restoring conversations indexes…' + ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true + end + + def deduplicate_custom_emojis! + remove_index_if_exists!(:custom_emojis, 'index_custom_emojis_on_shortcode_and_domain') + + @prompt.say 'Deduplicating custom_emojis…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emojis GROUP BY shortcode, domain HAVING count(*) > 1").each do |row| + emojis = CustomEmoji.where(id: row['ids'].split(',')).sort_by(&:id).reverse + + ref_emoji = emojis.shift + + emojis.each do |other| + merge_custom_emojis!(ref_emoji, other) + other.destroy + end + end + + @prompt.say 'Restoring custom_emojis indexes…' + ActiveRecord::Base.connection.add_index :custom_emojis, ['shortcode', 'domain'], name: 'index_custom_emojis_on_shortcode_and_domain', unique: true + end + + def deduplicate_custom_emoji_categories! + remove_index_if_exists!(:custom_emoji_categories, 'index_custom_emoji_categories_on_name') + + @prompt.say 'Deduplicating custom_emoji_categories…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emoji_categories GROUP BY name HAVING count(*) > 1").each do |row| + categories = CustomEmojiCategory.where(id: row['ids'].split(',')).sort_by(&:id).reverse + + ref_category = categories.shift + + categories.each do |other| + merge_custom_emoji_categories!(ref_category, other) + other.destroy + end + end + + @prompt.say 'Restoring custom_emoji_categories indexes…' + ActiveRecord::Base.connection.add_index :custom_emoji_categories, ['name'], name: 'index_custom_emoji_categories_on_name', unique: true + end + + def deduplicate_domain_allows! + remove_index_if_exists!(:domain_allows, 'index_domain_allows_on_domain') + + @prompt.say 'Deduplicating domain_allows…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_allows GROUP BY domain HAVING count(*) > 1").each do |row| + DomainAllow.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring domain_allows indexes…' + ActiveRecord::Base.connection.add_index :domain_allows, ['domain'], name: 'index_domain_allows_on_domain', unique: true + end + + def deduplicate_domain_blocks! + remove_index_if_exists!(:domain_blocks, 'index_domain_blocks_on_domain') + + @prompt.say 'Deduplicating domain_allows…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row| + domain_blocks = DomainBlock.where(id: row['ids'].split(',')).by_severity.reverse.to_a + + reject_media = domain_blocks.any?(&:reject_media?) + reject_reports = domain_blocks.any?(&:reject_reports?) + + reference_block = domain_blocks.shift + + private_comment = domain_blocks.reduce(reference_block.private_comment.presence) { |a, b| a || b.private_comment.presence } + public_comment = domain_blocks.reduce(reference_block.public_comment.presence) { |a, b| a || b.public_comment.presence } + + reference_block.update!(reject_media: reject_media, reject_reports: reject_reports, private_comment: private_comment, public_comment: public_comment) + + domain_blocks.each(&:destroy) + end + + @prompt.say 'Restoring domain_blocks indexes…' + ActiveRecord::Base.connection.add_index :domain_blocks, ['domain'], name: 'index_domain_blocks_on_domain', unique: true + end + + def deduplicate_unavailable_domains! + return unless ActiveRecord::Base.connection.table_exists?(:unavailable_domains) + + remove_index_if_exists!(:unavailable_domains, 'index_unavailable_domains_on_domain') + + @prompt.say 'Deduplicating unavailable_domains…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM unavailable_domains GROUP BY domain HAVING count(*) > 1").each do |row| + UnavailableDomain.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring domain_allows indexes…' + ActiveRecord::Base.connection.add_index :unavailable_domains, ['domain'], name: 'index_unavailable_domains_on_domain', unique: true + end + + def deduplicate_email_domain_blocks! + remove_index_if_exists!(:email_domain_blocks, 'index_email_domain_blocks_on_domain') + + @prompt.say 'Deduplicating email_domain_blocks…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM email_domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row| + domain_blocks = EmailDomainBlock.where(id: row['ids'].split(',')).sort_by { |b| b.parent.nil? ? 1 : 0 }.to_a + domain_blocks.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring email_domain_blocks indexes…' + ActiveRecord::Base.connection.add_index :email_domain_blocks, ['domain'], name: 'index_email_domain_blocks_on_domain', unique: true + end + + def deduplicate_media_attachments! + remove_index_if_exists!(:media_attachments, 'index_media_attachments_on_shortcode') + + @prompt.say 'Deduplicating media_attachments…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM media_attachments WHERE shortcode IS NOT NULL GROUP BY shortcode HAVING count(*) > 1").each do |row| + MediaAttachment.where(id: row['ids'].split(',').drop(1)).update_all(shortcode: nil) + end + + @prompt.say 'Restoring media_attachments indexes…' + ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true + end + + def deduplicate_preview_cards! + remove_index_if_exists!(:preview_cards, 'index_preview_cards_on_url') + + @prompt.say 'Deduplicating preview_cards…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM preview_cards GROUP BY url HAVING count(*) > 1").each do |row| + PreviewCard.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring preview_cards indexes…' + ActiveRecord::Base.connection.add_index :preview_cards, ['url'], name: 'index_preview_cards_on_url', unique: true + end + + def deduplicate_statuses! + remove_index_if_exists!(:statuses, 'index_statuses_on_uri') + + @prompt.say 'Deduplicating statuses…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM statuses WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row| + statuses = Status.where(id: row['ids'].split(',')).sort_by(&:id) + ref_status = statuses.shift + statuses.each do |status| + merge_statuses!(ref_status, status) if status.account_id == ref_status.account_id + status.destroy + end + end + + @prompt.say 'Restoring statuses indexes…' + ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true + end + + def deduplicate_tags! + remove_index_if_exists!(:tags, 'index_tags_on_name_lower') + + @prompt.say 'Deduplicating tags…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM tags GROUP BY lower((name)::text) HAVING count(*) > 1").each do |row| + tags = Tag.where(id: row['ids'].split(',')).sort_by { |t| [t.usable?, t.trendable?, t.listable?].count(false) } + ref_tag = tags.shift + tags.each do |tag| + merge_tags!(ref_tag, tag) + tag.destroy + end + end + + @prompt.say 'Restoring tags indexes…' + ActiveRecord::Base.connection.add_index :tags, 'lower((name)::text)', name: 'index_tags_on_name_lower', unique: true + end + + def deduplicate_webauthn_credentials! + return unless ActiveRecord::Base.connection.table_exists?(:webauthn_credentials) + + remove_index_if_exists!(:webauthn_credentials, 'index_webauthn_credentials_on_external_id') + + @prompt.say 'Deduplicating webauthn_credentials…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webauthn_credentials GROUP BY external_id HAVING count(*) > 1").each do |row| + WebauthnCredential.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring webauthn_credentials indexes…' + ActiveRecord::Base.connection.add_index :webauthn_credentials, ['external_id'], name: 'index_webauthn_credentials_on_external_id', unique: true + end + + def deduplicate_local_accounts!(accounts) + accounts = accounts.sort_by(&:id).reverse + + @prompt.warn "Multiple local accounts were found for username '#{accounts.first.username}'." + @prompt.warn 'All those accounts are distinct accounts but only the most recently-created one is fully-functionnal.' + + accounts.each_with_index do |account, idx| + @prompt.say '%2d. %s: created at: %s; updated at: %s; last logged in at: %s; statuses: %5d; last status at: %s' % [idx, account.username, account.created_at, account.updated_at, account.user&.last_sign_in_at&.to_s || 'N/A', account.account_stat&.statuses_count || 0, account.account_stat&.last_status_at || 'N/A'] + end + + @prompt.say 'Please chose the one to keep unchanged, other ones will be automatically renamed.' + + ref_id = @prompt.ask('Account to keep unchanged:') do |q| + q.required true + q.default 0 + q.convert :int + end + + accounts.delete_at(ref_id) + + i = 0 + accounts.each do |account| + i += 1 + username = account.username + "_#{i}" + + while Account.local.exists?(username: username) + i += 1 + username = account.username + "_#{i}" + end + + account.update!(username: username) + end + end + + def deduplicate_remote_accounts!(accounts) + accounts = accounts.sort_by(&:updated_at).reverse + + reference_account = accounts.shift + + accounts.each do |other_account| + if other_account.public_key == reference_account.public_key + # The accounts definitely point to the same resource, so + # it's safe to re-attribute content and relationships + reference_account.merge_with!(other_account) + end + + other_account.destroy + end + end + + def merge_conversations!(main_conv, duplicate_conv) + owned_classes = [ConversationMute, AccountConversation] + owned_classes.each do |klass| + klass.where(conversation_id: duplicate_conv.id).find_each do |record| + begin + record.update_attribute(:account_id, main_conv.id) + rescue ActiveRecord::RecordNotUnique + next + end + end + end + end + + def merge_custom_emojis!(main_emoji, duplicate_emoji) + owned_classes = [AnnouncementReaction] + owned_classes.each do |klass| + klass.where(custom_emoji_id: duplicate_emoji.id).update_all(custom_emoji_id: main_emoji.id) + end + end + + def merge_custom_emoji_categories!(main_category, duplicate_category) + owned_classes = [CustomEmoji] + owned_classes.each do |klass| + klass.where(category_id: duplicate_category.id).update_all(category_id: main_category.id) + end + end + + def merge_statuses!(main_status, duplicate_status) + owned_classes = [Favourite, Mention, Poll] + owned_classes << Bookmark if ActiveRecord::Base.connection.table_exists?(:bookmarks) + owned_classes.each do |klass| + klass.where(status_id: duplicate_status.id).find_each do |record| + begin + record.update_attribute(:status_id, main_status.id) + rescue ActiveRecord::RecordNotUnique + next + end + end + end + + StatusPin.where(account_id: main_status.account_id, status_id: duplicate_status.id).find_each do |record| + begin + record.update_attribute(:status_id, main_status.id) + rescue ActiveRecord::RecordNotUnique + next + end + end + + Status.where(in_reply_to_id: duplicate_status.id).find_each do |record| + begin + record.update_attribute(:in_reply_to_id, main_status.id) + rescue ActiveRecord::RecordNotUnique + next + end + end + + Status.where(reblog_of_id: duplicate_status.id).find_each do |record| + begin + record.update_attribute(:reblog_of_id, main_status.id) + rescue ActiveRecord::RecordNotUnique + next + end + end + end + + def merge_tags!(main_tag, duplicate_tag) + [FeaturedTag].each do |klass| + klass.where(tag_id: duplicate_tag.id).find_each do |record| + begin + record.update_attribute(:tag_id, main_tag.id) + rescue ActiveRecord::RecordNotUnique + next + end + end + end + end + + def find_duplicate_accounts + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM accounts GROUP BY lower(username), COALESCE(lower(domain), '') HAVING count(*) > 1") + end + + def remove_index_if_exists!(table, name) + ActiveRecord::Base.connection.remove_index(table, name: name) + rescue ArgumentError + nil + rescue ActiveRecord::StatementInvalid + nil + end + end +end diff --git a/lib/paperclip/response_with_limit_adapter.rb b/lib/paperclip/response_with_limit_adapter.rb index 8711b1349..17a2abd25 100644 --- a/lib/paperclip/response_with_limit_adapter.rb +++ b/lib/paperclip/response_with_limit_adapter.rb @@ -16,7 +16,7 @@ module Paperclip private def cache_current_values - @original_filename = filename_from_content_disposition || filename_from_path || 'data' + @original_filename = filename_from_content_disposition.presence || filename_from_path.presence || 'data' @size = @target.response.content_length @tempfile = copy_to_tempfile(@target) @content_type = ContentTypeDetector.new(@tempfile.path).detect diff --git a/spec/controllers/auth/sessions_controller_spec.rb b/spec/controllers/auth/sessions_controller_spec.rb index c387842cd..4560d64a8 100644 --- a/spec/controllers/auth/sessions_controller_spec.rb +++ b/spec/controllers/auth/sessions_controller_spec.rb @@ -215,7 +215,7 @@ RSpec.describe Auth::SessionsController, type: :controller do context 'using a valid OTP' do before do - post :create, params: { user: { otp_attempt: user.current_otp } }, session: { attempt_user_id: user.id } + post :create, params: { user: { otp_attempt: user.current_otp } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s } end it 'redirects to home' do @@ -230,7 +230,7 @@ RSpec.describe Auth::SessionsController, type: :controller do context 'when the server has an decryption error' do before do allow_any_instance_of(User).to receive(:validate_and_consume_otp!).and_raise(OpenSSL::Cipher::CipherError) - post :create, params: { user: { otp_attempt: user.current_otp } }, session: { attempt_user_id: user.id } + post :create, params: { user: { otp_attempt: user.current_otp } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s } end it 'shows a login error' do @@ -244,7 +244,7 @@ RSpec.describe Auth::SessionsController, type: :controller do context 'using a valid recovery code' do before do - post :create, params: { user: { otp_attempt: recovery_codes.first } }, session: { attempt_user_id: user.id } + post :create, params: { user: { otp_attempt: recovery_codes.first } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s } end it 'redirects to home' do @@ -258,7 +258,7 @@ RSpec.describe Auth::SessionsController, type: :controller do context 'using an invalid OTP' do before do - post :create, params: { user: { otp_attempt: 'wrongotp' } }, session: { attempt_user_id: user.id } + post :create, params: { user: { otp_attempt: 'wrongotp' } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s } end it 'shows a login error' do @@ -302,7 +302,7 @@ RSpec.describe Auth::SessionsController, type: :controller do context 'using a valid sign in token' do before do user.generate_sign_in_token && user.save - post :create, params: { user: { sign_in_token_attempt: user.sign_in_token } }, session: { attempt_user_id: user.id } + post :create, params: { user: { sign_in_token_attempt: user.sign_in_token } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s } end it 'redirects to home' do @@ -316,7 +316,7 @@ RSpec.describe Auth::SessionsController, type: :controller do context 'using an invalid sign in token' do before do - post :create, params: { user: { sign_in_token_attempt: 'wrongotp' } }, session: { attempt_user_id: user.id } + post :create, params: { user: { sign_in_token_attempt: 'wrongotp' } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s } end it 'shows a login error' do diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb index 98d29e6f3..75f628076 100644 --- a/spec/models/account_spec.rb +++ b/spec/models/account_spec.rb @@ -817,4 +817,27 @@ RSpec.describe Account, type: :model do include_examples 'AccountAvatar', :account include_examples 'AccountHeader', :account + + describe '#increment_count!' do + subject { Fabricate(:account) } + + it 'increments the count in multi-threaded an environment when account_stat is not yet initialized' do + subject + + increment_by = 15 + wait_for_start = true + + threads = Array.new(increment_by) do + Thread.new do + true while wait_for_start + Account.find(subject.id).increment_count!(:followers_count) + end + end + + wait_for_start = false + threads.each(&:join) + + expect(subject.reload.followers_count).to eq 15 + end + end end diff --git a/spec/services/resolve_account_service_spec.rb b/spec/services/resolve_account_service_spec.rb index cea942e39..6e4a0d9fe 100644 --- a/spec/services/resolve_account_service_spec.rb +++ b/spec/services/resolve_account_service_spec.rb @@ -4,23 +4,61 @@ RSpec.describe ResolveAccountService, type: :service do subject { described_class.new } before do - stub_request(:get, "https://quitter.no/.well-known/host-meta").to_return(request_fixture('.host-meta.txt')) - stub_request(:get, "https://example.com/.well-known/webfinger?resource=acct:catsrgr8@example.com").to_return(status: 404) stub_request(:get, "https://example.com/.well-known/host-meta").to_return(status: 404) stub_request(:get, "https://quitter.no/avatar/7477-300-20160211190340.png").to_return(request_fixture('avatar.txt')) - stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:catsrgr8@quitter.no").to_return(status: 404) stub_request(:get, "https://ap.example.com/.well-known/webfinger?resource=acct:foo@ap.example.com").to_return(request_fixture('activitypub-webfinger.txt')) stub_request(:get, "https://ap.example.com/users/foo").to_return(request_fixture('activitypub-actor.txt')) stub_request(:get, "https://ap.example.com/users/foo.atom").to_return(request_fixture('activitypub-feed.txt')) stub_request(:get, %r{https://ap.example.com/users/foo/\w+}).to_return(status: 404) end - it 'raises error if no such user can be resolved via webfinger' do - expect(subject.call('catsrgr8@quitter.no')).to be_nil + context 'when there is an LRDD endpoint but no resolvable account' do + before do + stub_request(:get, "https://quitter.no/.well-known/host-meta").to_return(request_fixture('.host-meta.txt')) + stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:catsrgr8@quitter.no").to_return(status: 404) + end + + it 'returns nil' do + expect(subject.call('catsrgr8@quitter.no')).to be_nil + end end - it 'raises error if the domain does not have webfinger' do - expect(subject.call('catsrgr8@example.com')).to be_nil + context 'when there is no LRDD endpoint nor resolvable account' do + before do + stub_request(:get, "https://example.com/.well-known/webfinger?resource=acct:catsrgr8@example.com").to_return(status: 404) + end + + it 'returns nil' do + expect(subject.call('catsrgr8@example.com')).to be_nil + end + end + + context 'with a legitimate webfinger redirection' do + before do + webfinger = { subject: 'acct:foo@ap.example.com', links: [{ rel: 'self', href: 'https://ap.example.com/users/foo' }] } + stub_request(:get, 'https://redirected.example.com/.well-known/webfinger?resource=acct:Foo@redirected.example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' }) + end + + it 'returns new remote account' do + account = subject.call('Foo@redirected.example.com') + + expect(account.activitypub?).to eq true + expect(account.acct).to eq 'foo@ap.example.com' + expect(account.inbox_url).to eq 'https://ap.example.com/users/foo/inbox' + end + end + + context 'with too many webfinger redirections' do + before do + webfinger = { subject: 'acct:foo@evil.example.com', links: [{ rel: 'self', href: 'https://ap.example.com/users/foo' }] } + stub_request(:get, 'https://redirected.example.com/.well-known/webfinger?resource=acct:Foo@redirected.example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' }) + webfinger2 = { subject: 'acct:foo@ap.example.com', links: [{ rel: 'self', href: 'https://ap.example.com/users/foo' }] } + stub_request(:get, 'https://evil.example.com/.well-known/webfinger?resource=acct:foo@evil.example.com').to_return(body: Oj.dump(webfinger2), headers: { 'Content-Type': 'application/jrd+json' }) + end + + it 'returns nil' do + expect(subject.call('Foo@redirected.example.com')).to be_nil + end end context 'with an ActivityPub account' do From 36b9b8deaa252b458d2fa6a3c9b31cb82b8dfedb Mon Sep 17 00:00:00 2001 From: ThibG Date: Sat, 19 Dec 2020 00:26:53 +0100 Subject: [PATCH 22/23] Fix ResolveAccountService accepting mismatching acct: URI (#15368) Co-authored-by: Claire --- app/services/resolve_account_service.rb | 10 +++------ spec/services/resolve_account_service_spec.rb | 21 ++++++++++++++++--- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb index eee1de51a..f8d61611f 100644 --- a/app/services/resolve_account_service.rb +++ b/app/services/resolve_account_service.rb @@ -46,7 +46,7 @@ class ResolveAccountService < BaseService # Now it is certain, it is definitely a remote account, and it # either needs to be created, or updated from fresh data - process_account! + fetch_account! rescue Webfinger::Error, WebfingerRedirectError, Oj::ParseError => e Rails.logger.debug "Webfinger query for #{@uri} failed: #{e}" nil @@ -99,16 +99,12 @@ class ResolveAccountService < BaseService acct.gsub(/\Aacct:/, '').split('@') end - def process_account! + def fetch_account! return unless activitypub_ready? RedisLock.acquire(lock_options) do |lock| if lock.acquired? - @account = Account.find_remote(@username, @domain) - - next if actor_json.nil? - - @account = ActivityPub::ProcessAccountService.new.call(@username, @domain, actor_json) + @account = ActivityPub::FetchRemoteAccountService.new.call(actor_url) else raise Mastodon::RaceConditionError end diff --git a/spec/services/resolve_account_service_spec.rb b/spec/services/resolve_account_service_spec.rb index 6e4a0d9fe..92c837050 100644 --- a/spec/services/resolve_account_service_spec.rb +++ b/spec/services/resolve_account_service_spec.rb @@ -35,7 +35,22 @@ RSpec.describe ResolveAccountService, type: :service do context 'with a legitimate webfinger redirection' do before do - webfinger = { subject: 'acct:foo@ap.example.com', links: [{ rel: 'self', href: 'https://ap.example.com/users/foo' }] } + webfinger = { subject: 'acct:foo@ap.example.com', links: [{ rel: 'self', href: 'https://ap.example.com/users/foo', type: 'application/activity+json' }] } + stub_request(:get, 'https://redirected.example.com/.well-known/webfinger?resource=acct:Foo@redirected.example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' }) + end + + it 'returns new remote account' do + account = subject.call('Foo@redirected.example.com') + + expect(account.activitypub?).to eq true + expect(account.acct).to eq 'foo@ap.example.com' + expect(account.inbox_url).to eq 'https://ap.example.com/users/foo/inbox' + end + end + + context 'with a misconfigured redirection' do + before do + webfinger = { subject: 'acct:Foo@redirected.example.com', links: [{ rel: 'self', href: 'https://ap.example.com/users/foo', type: 'application/activity+json' }] } stub_request(:get, 'https://redirected.example.com/.well-known/webfinger?resource=acct:Foo@redirected.example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' }) end @@ -50,9 +65,9 @@ RSpec.describe ResolveAccountService, type: :service do context 'with too many webfinger redirections' do before do - webfinger = { subject: 'acct:foo@evil.example.com', links: [{ rel: 'self', href: 'https://ap.example.com/users/foo' }] } + webfinger = { subject: 'acct:foo@evil.example.com', links: [{ rel: 'self', href: 'https://ap.example.com/users/foo', type: 'application/activity+json' }] } stub_request(:get, 'https://redirected.example.com/.well-known/webfinger?resource=acct:Foo@redirected.example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' }) - webfinger2 = { subject: 'acct:foo@ap.example.com', links: [{ rel: 'self', href: 'https://ap.example.com/users/foo' }] } + webfinger2 = { subject: 'acct:foo@ap.example.com', links: [{ rel: 'self', href: 'https://ap.example.com/users/foo', type: 'application/activity+json' }] } stub_request(:get, 'https://evil.example.com/.well-known/webfinger?resource=acct:foo@evil.example.com').to_return(body: Oj.dump(webfinger2), headers: { 'Content-Type': 'application/jrd+json' }) end From 2d5a5bac673c5d8fec78be865c10ad6ae5a958a5 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 19 Dec 2020 00:41:49 +0100 Subject: [PATCH 23/23] Bump version to 3.2.2 --- CHANGELOG.md | 26 ++++++++++++++++++++++++++ chart/Chart.yaml | 4 ++-- chart/values.yaml.template | 2 +- lib/mastodon/version.rb | 2 +- 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f94ebea2..cfc95dcf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,32 @@ Changelog All notable changes to this project will be documented in this file. +## [3.2.2] - 2020-12-19 +### Added + +- Add `tootctl maintenance fix-duplicates` ([ThibG](https://github.com/tootsuite/mastodon/pull/14860), [Gargron](https://github.com/tootsuite/mastodon/pull/15223)) + - Index corruption in the database? + - This command is for you + +### Removed + +- Remove dependency on unused and unmaintained http_parser.rb gem ([ThibG](https://github.com/tootsuite/mastodon/pull/14574)) + +### Fixed + +- Fix Move handler not being triggered when failing to fetch target account ([ThibG](https://github.com/tootsuite/mastodon/pull/15107)) +- Fix downloading remote media files when server returns empty filename ([ThibG](https://github.com/tootsuite/mastodon/pull/14867)) +- Fix possible casing inconsistencies in hashtag search ([ThibG](https://github.com/tootsuite/mastodon/pull/14906)) +- Fix updating account counters when association is not yet created ([Gargron](https://github.com/tootsuite/mastodon/pull/15108)) +- Fix account processing failing because of large collections ([ThibG](https://github.com/tootsuite/mastodon/pull/15027)) +- Fix resolving an account through its non-canonical form (i.e. alternate domain) ([ThibG](https://github.com/tootsuite/mastodon/pull/15187)) +- Fix slow distinct queries where grouped queries are faster ([Gargron](https://github.com/tootsuite/mastodon/pull/15287)) + +### Security + +- Fix 2FA/sign-in token sessions being valid after password change ([Gargron](https://github.com/tootsuite/mastodon/pull/14802)) +- Fix resolving accounts sometimes creating duplicate records for a given ActivityPub identifier ([ThibG](https://github.com/tootsuite/mastodon/pull/15364)) + ## [3.2.1] - 2020-10-19 ### Added diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 783569451..4144ba8b1 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -15,12 +15,12 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.1.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: 3.1.5 +appVersion: 3.2.2 dependencies: - name: elasticsearch diff --git a/chart/values.yaml.template b/chart/values.yaml.template index 694bc4d42..e1984068c 100644 --- a/chart/values.yaml.template +++ b/chart/values.yaml.template @@ -4,7 +4,7 @@ image: repository: tootsuite/mastodon pullPolicy: Always # https://hub.docker.com/r/tootsuite/mastodon/tags - tag: v3.1.5 + tag: v3.2.2 # alternatively, use `latest` for the latest release or `edge` for the image # built from the most recent commit # diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 344ea996e..89cedbb87 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 1 + 2 end def flags