diff --git a/CHANGELOG.md b/CHANGELOG.md index 18790e860..cfc95dcf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,61 @@ Changelog All notable changes to this project will be documented in this file. +## [3.2.2] - 2020-12-19 +### Added + +- Add `tootctl maintenance fix-duplicates` ([ThibG](https://github.com/tootsuite/mastodon/pull/14860), [Gargron](https://github.com/tootsuite/mastodon/pull/15223)) + - Index corruption in the database? + - This command is for you + +### Removed + +- Remove dependency on unused and unmaintained http_parser.rb gem ([ThibG](https://github.com/tootsuite/mastodon/pull/14574)) + +### Fixed + +- Fix Move handler not being triggered when failing to fetch target account ([ThibG](https://github.com/tootsuite/mastodon/pull/15107)) +- Fix downloading remote media files when server returns empty filename ([ThibG](https://github.com/tootsuite/mastodon/pull/14867)) +- Fix possible casing inconsistencies in hashtag search ([ThibG](https://github.com/tootsuite/mastodon/pull/14906)) +- Fix updating account counters when association is not yet created ([Gargron](https://github.com/tootsuite/mastodon/pull/15108)) +- Fix account processing failing because of large collections ([ThibG](https://github.com/tootsuite/mastodon/pull/15027)) +- Fix resolving an account through its non-canonical form (i.e. alternate domain) ([ThibG](https://github.com/tootsuite/mastodon/pull/15187)) +- Fix slow distinct queries where grouped queries are faster ([Gargron](https://github.com/tootsuite/mastodon/pull/15287)) + +### Security + +- Fix 2FA/sign-in token sessions being valid after password change ([Gargron](https://github.com/tootsuite/mastodon/pull/14802)) +- Fix resolving accounts sometimes creating duplicate records for a given ActivityPub identifier ([ThibG](https://github.com/tootsuite/mastodon/pull/15364)) + +## [3.2.1] - 2020-10-19 +### Added + +- Add support for latest HTTP Signatures spec draft ([ThibG](https://github.com/tootsuite/mastodon/pull/14556)) +- Add support for inlined objects in ActivityPub `to`/`cc` ([ThibG](https://github.com/tootsuite/mastodon/pull/14514)) + +### Changed + +- Change actors to not be served at all without authentication in limited federation mode ([ThibG](https://github.com/tootsuite/mastodon/pull/14800)) + - Previously, a bare version of an actor was served when not authenticated, i.e. username and public key + - Because all actor fetch requests are signed using a separate system actor, that is no longer required + +### Fixed + +- Fix `tootctl media` commands not recognizing very large IDs ([ThibG](https://github.com/tootsuite/mastodon/pull/14536)) +- Fix crash when failing to load emoji picker in web UI ([ThibG](https://github.com/tootsuite/mastodon/pull/14525)) +- Fix contrast requirements in thumbnail color extraction ([ThibG](https://github.com/tootsuite/mastodon/pull/14464)) +- Fix audio/video player not using `CDN_HOST` on public pages ([ThibG](https://github.com/tootsuite/mastodon/pull/14486)) +- Fix private boost icon not being used on public pages ([OmmyZhang](https://github.com/tootsuite/mastodon/pull/14471)) +- Fix audio player on Safari in web UI ([ThibG](https://github.com/tootsuite/mastodon/pull/14485), [ThibG](https://github.com/tootsuite/mastodon/pull/14465)) +- Fix dereferencing remote statuses not using the correct account for signature when receiving a targeted inbox delivery ([ThibG](https://github.com/tootsuite/mastodon/pull/14656)) +- Fix nil error in `tootctl media remove` ([noellabo](https://github.com/tootsuite/mastodon/pull/14657)) +- Fix videos with near-60 fps being rejected ([Gargron](https://github.com/tootsuite/mastodon/pull/14684)) +- Fix reported statuses not being included in warning e-mail ([Gargron](https://github.com/tootsuite/mastodon/pull/14778)) +- Fix `Reject` activities of `Follow` objects not correctly destroying a follow relationship ([ThibG](https://github.com/tootsuite/mastodon/pull/14479)) +- Fix inefficiencies in fan-out-on-write service ([Gargron](https://github.com/tootsuite/mastodon/pull/14682), [noellabo](https://github.com/tootsuite/mastodon/pull/14709)) +- Fix timeout errors when trying to webfinger some IPv6 configurations ([Gargron](https://github.com/tootsuite/mastodon/pull/14919)) +- Fix files served as `application/octet-stream` being rejected without attempting mime type detection ([ThibG](https://github.com/tootsuite/mastodon/pull/14452)) + ## [3.2.0] - 2020-07-27 ### Added diff --git a/Gemfile b/Gemfile index 414bd2c30..5ac1c79c1 100644 --- a/Gemfile +++ b/Gemfile @@ -54,14 +54,12 @@ gem 'doorkeeper', '~> 5.4' gem 'ed25519', '~> 1.2' gem 'fast_blank', '~> 1.0' gem 'fastimage' -gem 'goldfinger', '~> 2.1' gem 'hiredis', '~> 0.6' gem 'redis-namespace', '~> 1.7' gem 'health_check', git: 'https://github.com/ianheggie/health_check', ref: '0b799ead604f900ed50685e9b2d469cd2befba5b' gem 'htmlentities', '~> 4.3' gem 'http', '~> 4.4' gem 'http_accept_language', '~> 2.1' -gem 'http_parser.rb', '~> 0.6', git: 'https://github.com/tmm1/http_parser.rb', ref: '54b17ba8c7d8d20a16dfc65d1775241833219cf2', submodules: true gem 'httplog', '~> 1.4.3' gem 'idn-ruby', require: 'idn' gem 'kaminari', '~> 1.2' diff --git a/Gemfile.lock b/Gemfile.lock index 3d4fce643..c3216870d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -13,14 +13,6 @@ GIT specs: posix-spawn (0.3.13) -GIT - remote: https://github.com/tmm1/http_parser.rb - revision: 54b17ba8c7d8d20a16dfc65d1775241833219cf2 - ref: 54b17ba8c7d8d20a16dfc65d1775241833219cf2 - submodules: true - specs: - http_parser.rb (0.6.1) - GIT remote: https://github.com/witgo/nilsimsa revision: fd184883048b922b176939f851338d0a4971a532 @@ -250,11 +242,6 @@ GEM ruby-progressbar (~> 1.4) globalid (0.4.2) activesupport (>= 4.2.0) - goldfinger (2.1.1) - addressable (~> 2.5) - http (~> 4.0) - nokogiri (~> 1.8) - oj (~> 3.0) hamlit (2.11.0) temple (>= 0.8.2) thor @@ -708,14 +695,12 @@ DEPENDENCIES fog-core (<= 2.1.0) fog-openstack (~> 0.3) fuubar (~> 2.5) - goldfinger (~> 2.1) hamlit-rails (~> 0.2) health_check! hiredis (~> 0.6) htmlentities (~> 4.3) http (~> 4.4) http_accept_language (~> 2.1) - http_parser.rb (~> 0.6)! httplog (~> 1.4.3) i18n-tasks (~> 0.9) idn-ruby diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index db77b628c..b42f711f0 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -7,6 +7,7 @@ class AccountsController < ApplicationController include AccountControllerConcern include SignatureAuthentication + before_action :require_signature!, if: -> { request.format == :json && authorized_fetch_mode? } before_action :set_cache_headers before_action :set_body_classes @@ -49,7 +50,7 @@ class AccountsController < ApplicationController format.json do expires_in 3.minutes, public: !(authorized_fetch_mode? && signed_request_account.present?) - render_with_cache json: @account, content_type: 'application/activity+json', serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter, fields: restrict_fields_to + render_with_cache json: @account, content_type: 'application/activity+json', serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter end end end @@ -81,7 +82,7 @@ class AccountsController < ApplicationController end def account_media_status_ids - @account.media_attachments.attached.reorder(nil).select(:status_id).distinct + @account.media_attachments.attached.reorder(nil).select(:status_id).group(:status_id) end def no_replies_scope @@ -149,12 +150,4 @@ class AccountsController < ApplicationController def params_slice(*keys) params.slice(*keys).permit(*keys) end - - def restrict_fields_to - if signed_request_account.present? || public_fetch_mode? - # Return all fields - else - %i(id type preferred_username inbox public_key endpoints) - end - end end diff --git a/app/controllers/admin/statuses_controller.rb b/app/controllers/admin/statuses_controller.rb index 650195034..d7c192f0d 100644 --- a/app/controllers/admin/statuses_controller.rb +++ b/app/controllers/admin/statuses_controller.rb @@ -14,7 +14,7 @@ module Admin @statuses = @account.statuses.where(visibility: [:public, :unlisted]) if params[:media] - account_media_status_ids = @account.media_attachments.attached.reorder(nil).select(:status_id).distinct + account_media_status_ids = @account.media_attachments.attached.reorder(nil).select(:status_id).group(:status_id) @statuses.merge!(Status.where(id: account_media_status_ids)) end diff --git a/app/controllers/api/base_controller.rb b/app/controllers/api/base_controller.rb index 045e7dd26..055cd416d 100644 --- a/app/controllers/api/base_controller.rb +++ b/app/controllers/api/base_controller.rb @@ -102,7 +102,7 @@ class Api::BaseController < ApplicationController elsif !current_user.approved? render json: { error: 'Your login is currently pending approval' }, status: 403 else - set_user_activity + update_user_sign_in end end diff --git a/app/controllers/auth/sessions_controller.rb b/app/controllers/auth/sessions_controller.rb index 1fd755334..783338450 100644 --- a/app/controllers/auth/sessions_controller.rb +++ b/app/controllers/auth/sessions_controller.rb @@ -7,6 +7,7 @@ class Auth::SessionsController < Devise::SessionsController skip_before_action :require_no_authentication, only: [:create] skip_before_action :require_functional! + skip_before_action :update_user_sign_in include TwoFactorAuthenticationConcern include SignInTokenAuthenticationConcern @@ -24,6 +25,7 @@ class Auth::SessionsController < Devise::SessionsController def create super do |resource| + resource.update_sign_in!(request, new_sign_in: true) remember_me(resource) flash.delete(:notice) end @@ -41,7 +43,7 @@ class Auth::SessionsController < Devise::SessionsController def find_user if session[:attempt_user_id] - User.find(session[:attempt_user_id]) + User.find_by(id: session[:attempt_user_id]) else user = User.authenticate_with_ldap(user_params) if Devise.ldap_authentication user ||= User.authenticate_with_pam(user_params) if Devise.pam_authentication @@ -74,6 +76,7 @@ class Auth::SessionsController < Devise::SessionsController def require_no_authentication super + # Delete flash message that isn't entirely useful and may be confusing in # most cases because /web doesn't display/clear flash messages. flash.delete(:alert) if flash[:alert] == I18n.t('devise.failure.already_authenticated') @@ -91,13 +94,30 @@ class Auth::SessionsController < Devise::SessionsController def home_paths(resource) paths = [about_path] + if single_user_mode? && resource.is_a?(User) paths << short_account_path(username: resource.account) end + paths end def continue_after? truthy_param?(:continue) end + + def restart_session + clear_attempt_from_session + redirect_to new_user_session_path, alert: I18n.t('devise.failure.timeout') + end + + def set_attempt_session(user) + session[:attempt_user_id] = user.id + session[:attempt_user_updated_at] = user.updated_at.to_s + end + + def clear_attempt_from_session + session.delete(:attempt_user_id) + session.delete(:attempt_user_updated_at) + end end diff --git a/app/controllers/concerns/sign_in_token_authentication_concern.rb b/app/controllers/concerns/sign_in_token_authentication_concern.rb index 91f813acc..3c95a4afd 100644 --- a/app/controllers/concerns/sign_in_token_authentication_concern.rb +++ b/app/controllers/concerns/sign_in_token_authentication_concern.rb @@ -18,7 +18,9 @@ module SignInTokenAuthenticationConcern def authenticate_with_sign_in_token user = self.resource = find_user - if user_params[:sign_in_token_attempt].present? && session[:attempt_user_id] + if user.present? && session[:attempt_user_id].present? && session[:attempt_user_updated_at] != user.updated_at.to_s + restart_session + elsif user_params.key?(:sign_in_token_attempt) && session[:attempt_user_id] authenticate_with_sign_in_token_attempt(user) elsif user.present? && user.external_or_valid_password?(user_params[:password]) prompt_for_sign_in_token(user) @@ -27,7 +29,7 @@ module SignInTokenAuthenticationConcern def authenticate_with_sign_in_token_attempt(user) if valid_sign_in_token_attempt?(user) - session.delete(:attempt_user_id) + clear_attempt_from_session remember_me(user) sign_in(user) else @@ -42,10 +44,10 @@ module SignInTokenAuthenticationConcern UserMailer.sign_in_token(user, request.remote_ip, request.user_agent, Time.now.utc.to_s).deliver_later! end - set_locale do - session[:attempt_user_id] = user.id - @body_classes = 'lighter' - render :sign_in_token - end + set_attempt_session(user) + + @body_classes = 'lighter' + + set_locale { render :sign_in_token } end end diff --git a/app/controllers/concerns/signature_verification.rb b/app/controllers/concerns/signature_verification.rb index 10efbf2e0..18f549de9 100644 --- a/app/controllers/concerns/signature_verification.rb +++ b/app/controllers/concerns/signature_verification.rb @@ -7,6 +7,44 @@ module SignatureVerification include DomainControlHelper + EXPIRATION_WINDOW_LIMIT = 12.hours + CLOCK_SKEW_MARGIN = 1.hour + + class SignatureVerificationError < StandardError; end + + class SignatureParamsParser < Parslet::Parser + rule(:token) { match("[0-9a-zA-Z!#$%&'*+.^_`|~-]").repeat(1).as(:token) } + rule(:quoted_string) { str('"') >> (qdtext | quoted_pair).repeat.as(:quoted_string) >> str('"') } + # qdtext and quoted_pair are not exactly according to spec but meh + rule(:qdtext) { match('[^\\\\"]') } + rule(:quoted_pair) { str('\\') >> any } + rule(:bws) { match('\s').repeat } + rule(:param) { (token.as(:key) >> bws >> str('=') >> bws >> (token | quoted_string).as(:value)).as(:param) } + rule(:comma) { bws >> str(',') >> bws } + # Old versions of node-http-signature add an incorrect "Signature " prefix to the header + rule(:buggy_prefix) { str('Signature ') } + rule(:params) { buggy_prefix.maybe >> (param >> (comma >> param).repeat).as(:params) } + root(:params) + end + + class SignatureParamsTransformer < Parslet::Transform + rule(params: subtree(:p)) do + (p.is_a?(Array) ? p : [p]).each_with_object({}) { |(key, val), h| h[key] = val } + end + + rule(param: { key: simple(:key), value: simple(:val) }) do + [key, val] + end + + rule(quoted_string: simple(:string)) do + string.to_s + end + + rule(token: simple(:string)) do + string.to_s + end + end + def require_signature! render plain: signature_verification_failure_reason, status: signature_verification_failure_code unless signed_request_account end @@ -24,72 +62,40 @@ module SignatureVerification end def signature_key_id - raw_signature = request.headers['Signature'] - signature_params = {} - - raw_signature.split(',').each do |part| - parsed_parts = part.match(/([a-z]+)="([^"]+)"/i) - next if parsed_parts.nil? || parsed_parts.size != 3 - signature_params[parsed_parts[1]] = parsed_parts[2] - end - signature_params['keyId'] + rescue SignatureVerificationError + nil end def signed_request_account return @signed_request_account if defined?(@signed_request_account) - unless signed_request? - @signature_verification_failure_reason = 'Request not signed' - @signed_request_account = nil - return - end + raise SignatureVerificationError, 'Request not signed' unless signed_request? + raise SignatureVerificationError, 'Incompatible request signature. keyId and signature are required' if missing_required_signature_parameters? + raise SignatureVerificationError, 'Unsupported signature algorithm (only rsa-sha256 and hs2019 are supported)' unless %w(rsa-sha256 hs2019).include?(signature_algorithm) + raise SignatureVerificationError, 'Signed request date outside acceptable time window' unless matches_time_window? - if request.headers['Date'].present? && !matches_time_window? - @signature_verification_failure_reason = 'Signed request date outside acceptable time window' - @signed_request_account = nil - return - end - - raw_signature = request.headers['Signature'] - signature_params = {} - - raw_signature.split(',').each do |part| - parsed_parts = part.match(/([a-z]+)="([^"]+)"/i) - next if parsed_parts.nil? || parsed_parts.size != 3 - signature_params[parsed_parts[1]] = parsed_parts[2] - end - - if incompatible_signature?(signature_params) - @signature_verification_failure_reason = 'Incompatible request signature' - @signed_request_account = nil - return - end + verify_signature_strength! account = account_from_key_id(signature_params['keyId']) - if account.nil? - @signature_verification_failure_reason = "Public key not found for key #{signature_params['keyId']}" - @signed_request_account = nil - return - end + raise SignatureVerificationError, "Public key not found for key #{signature_params['keyId']}" if account.nil? signature = Base64.decode64(signature_params['signature']) - compare_signed_string = build_signed_string(signature_params['headers']) + compare_signed_string = build_signed_string return account unless verify_signature(account, signature, compare_signed_string).nil? account = stoplight_wrap_request { account.possibly_stale? ? account.refresh! : account_refresh_key(account) } - if account.nil? - @signature_verification_failure_reason = "Public key not found for key #{signature_params['keyId']}" - @signed_request_account = nil - return - end + raise SignatureVerificationError, "Public key not found for key #{signature_params['keyId']}" if account.nil? return account unless verify_signature(account, signature, compare_signed_string).nil? - @signature_verification_failure_reason = "Verification failed for #{account.username}@#{account.domain} #{account.uri}" + @signature_verification_failure_reason = "Verification failed for #{account.username}@#{account.domain} #{account.uri} using rsa-sha256 (RSASSA-PKCS1-v1_5 with SHA-256)" + @signed_request_account = nil + rescue SignatureVerificationError => e + @signature_verification_failure_reason = e.message @signed_request_account = nil end @@ -99,6 +105,31 @@ module SignatureVerification private + def signature_params + @signature_params ||= begin + raw_signature = request.headers['Signature'] + tree = SignatureParamsParser.new.parse(raw_signature) + SignatureParamsTransformer.new.apply(tree) + end + rescue Parslet::ParseFailed + raise SignatureVerificationError, 'Error parsing signature parameters' + end + + def signature_algorithm + signature_params.fetch('algorithm', 'hs2019') + end + + def signed_headers + signature_params.fetch('headers', signature_algorithm == 'hs2019' ? '(created)' : 'date').downcase.split(' ') + end + + def verify_signature_strength! + raise SignatureVerificationError, 'Mastodon requires the Date header or (created) pseudo-header to be signed' unless signed_headers.include?('date') || signed_headers.include?('(created)') + raise SignatureVerificationError, 'Mastodon requires the Digest header or (request-target) pseudo-header to be signed' unless signed_headers.include?(Request::REQUEST_TARGET) || signed_headers.include?('digest') + raise SignatureVerificationError, 'Mastodon requires the Host header to be signed' unless signed_headers.include?('host') + raise SignatureVerificationError, 'Mastodon requires the Digest header to be signed when doing a POST request' if request.post? && !signed_headers.include?('digest') + end + def verify_signature(account, signature, compare_signed_string) if account.keypair.public_key.verify(OpenSSL::Digest::SHA256.new, signature, compare_signed_string) @signed_request_account = account @@ -108,12 +139,20 @@ module SignatureVerification nil end - def build_signed_string(signed_headers) - signed_headers = 'date' if signed_headers.blank? - - signed_headers.downcase.split(' ').map do |signed_header| + def build_signed_string + signed_headers.map do |signed_header| if signed_header == Request::REQUEST_TARGET "#{Request::REQUEST_TARGET}: #{request.method.downcase} #{request.path}" + elsif signed_header == '(created)' + raise SignatureVerificationError, 'Invalid pseudo-header (created) for rsa-sha256' unless signature_algorithm == 'hs2019' + raise SignatureVerificationError, 'Pseudo-header (created) used but corresponding argument missing' if signature_params['created'].blank? + + "(created): #{signature_params['created']}" + elsif signed_header == '(expires)' + raise SignatureVerificationError, 'Invalid pseudo-header (expires) for rsa-sha256' unless signature_algorithm == 'hs2019' + raise SignatureVerificationError, 'Pseudo-header (expires) used but corresponding argument missing' if signature_params['expires'].blank? + + "(expires): #{signature_params['expires']}" elsif signed_header == 'digest' "digest: #{body_digest}" else @@ -123,13 +162,28 @@ module SignatureVerification end def matches_time_window? + created_time = nil + expires_time = nil + begin - time_sent = Time.httpdate(request.headers['Date']) + if signature_algorithm == 'hs2019' && signature_params['created'].present? + created_time = Time.at(signature_params['created'].to_i).utc + elsif request.headers['Date'].present? + created_time = Time.httpdate(request.headers['Date']).utc + end + + expires_time = Time.at(signature_params['expires'].to_i).utc if signature_params['expires'].present? rescue ArgumentError return false end - (Time.now.utc - time_sent).abs <= 12.hours + expires_time ||= created_time + 5.minutes unless created_time.nil? + expires_time = [expires_time, created_time + EXPIRATION_WINDOW_LIMIT].min unless created_time.nil? + + return false if created_time.present? && created_time > Time.now.utc + CLOCK_SKEW_MARGIN + return false if expires_time.present? && Time.now.utc > expires_time + CLOCK_SKEW_MARGIN + + true end def body_digest @@ -140,9 +194,8 @@ module SignatureVerification name.split(/-/).map(&:capitalize).join('-') end - def incompatible_signature?(signature_params) - signature_params['keyId'].blank? || - signature_params['signature'].blank? + def missing_required_signature_parameters? + signature_params['keyId'].blank? || signature_params['signature'].blank? end def account_from_key_id(key_id) diff --git a/app/controllers/concerns/two_factor_authentication_concern.rb b/app/controllers/concerns/two_factor_authentication_concern.rb index daafe56f4..f54a3c829 100644 --- a/app/controllers/concerns/two_factor_authentication_concern.rb +++ b/app/controllers/concerns/two_factor_authentication_concern.rb @@ -21,7 +21,9 @@ module TwoFactorAuthenticationConcern def authenticate_with_two_factor user = self.resource = find_user - if user_params[:otp_attempt].present? && session[:attempt_user_id] + if user.present? && session[:attempt_user_id].present? && session[:attempt_user_updated_at] != user.updated_at.to_s + restart_session + elsif user_params[:otp_attempt].present? && session[:attempt_user_id] authenticate_with_two_factor_attempt(user) elsif user.present? && user.external_or_valid_password?(user_params[:password]) prompt_for_two_factor(user) @@ -30,7 +32,7 @@ module TwoFactorAuthenticationConcern def authenticate_with_two_factor_attempt(user) if valid_otp_attempt?(user) - session.delete(:attempt_user_id) + clear_attempt_from_session remember_me(user) sign_in(user) else @@ -40,10 +42,10 @@ module TwoFactorAuthenticationConcern end def prompt_for_two_factor(user) - set_locale do - session[:attempt_user_id] = user.id - @body_classes = 'lighter' - render :two_factor - end + set_attempt_session(user) + + @body_classes = 'lighter' + + set_locale { render :two_factor } end end diff --git a/app/controllers/concerns/user_tracking_concern.rb b/app/controllers/concerns/user_tracking_concern.rb index be10705fc..efda37fae 100644 --- a/app/controllers/concerns/user_tracking_concern.rb +++ b/app/controllers/concerns/user_tracking_concern.rb @@ -6,14 +6,13 @@ module UserTrackingConcern UPDATE_SIGN_IN_HOURS = 24 included do - before_action :set_user_activity + before_action :update_user_sign_in end private - def set_user_activity - return unless user_needs_sign_in_update? - current_user.update_tracked_fields!(request) + def update_user_sign_in + current_user.update_sign_in!(request) if user_needs_sign_in_update? end def user_needs_sign_in_update? diff --git a/app/helpers/webfinger_helper.rb b/app/helpers/webfinger_helper.rb index ab7ca4698..482f4e19e 100644 --- a/app/helpers/webfinger_helper.rb +++ b/app/helpers/webfinger_helper.rb @@ -1,38 +1,7 @@ # frozen_string_literal: true -# Monkey-patch on monkey-patch. -# Because it conflicts with the request.rb patch. -class HTTP::Timeout::PerOperationOriginal < HTTP::Timeout::PerOperation - def connect(socket_class, host, port, nodelay = false) - ::Timeout.timeout(@connect_timeout, HTTP::TimeoutError) do - @socket = socket_class.open(host, port) - @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay - end - end -end - module WebfingerHelper def webfinger!(uri) - hidden_service_uri = /\.(onion|i2p)(:\d+)?$/.match(uri) - - raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if !Rails.configuration.x.access_to_hidden_service && hidden_service_uri - - opts = { - ssl: !hidden_service_uri, - - headers: { - 'User-Agent': Mastodon::Version.user_agent, - }, - - timeout_class: HTTP::Timeout::PerOperationOriginal, - - timeout_options: { - write_timeout: 10, - connect_timeout: 5, - read_timeout: 10, - }, - } - - Goldfinger::Client.new(uri, opts.merge(Rails.configuration.x.http_client_proxy)).finger + Webfinger.new(uri).perform end end diff --git a/app/javascript/mastodon/components/status_action_bar.js b/app/javascript/mastodon/components/status_action_bar.js index 231c517e9..b47855a7e 100644 --- a/app/javascript/mastodon/components/status_action_bar.js +++ b/app/javascript/mastodon/components/status_action_bar.js @@ -7,6 +7,7 @@ import DropdownMenuContainer from '../containers/dropdown_menu_container'; import { defineMessages, injectIntl } from 'react-intl'; import ImmutablePureComponent from 'react-immutable-pure-component'; import { me, isStaff } from '../initial_state'; +import classNames from 'classnames'; const messages = defineMessages({ delete: { id: 'status.delete', defaultMessage: 'Delete' }, @@ -329,7 +330,7 @@ class StatusActionBar extends ImmutablePureComponent { return (
{obfuscatedCount(status.get('replies_count'))}
- + {shareButton} diff --git a/app/javascript/mastodon/features/audio/index.js b/app/javascript/mastodon/features/audio/index.js index 1ab1c3117..5b8172694 100644 --- a/app/javascript/mastodon/features/audio/index.js +++ b/app/javascript/mastodon/features/audio/index.js @@ -115,6 +115,10 @@ class Audio extends React.PureComponent { } togglePlay = () => { + if (!this.audioContext) { + this._initAudioContext(); + } + if (this.state.paused) { this.setState({ paused: false }, () => this.audio.play()); } else { @@ -133,10 +137,6 @@ class Audio extends React.PureComponent { handlePlay = () => { this.setState({ paused: false }); - if (this.canvas && !this.audioContext) { - this._initAudioContext(); - } - if (this.audioContext && this.audioContext.state === 'suspended') { this.audioContext.resume(); } @@ -269,8 +269,9 @@ class Audio extends React.PureComponent { } _initAudioContext () { - const context = new AudioContext(); - const source = context.createMediaElementSource(this.audio); + const AudioContext = window.AudioContext || window.webkitAudioContext; + const context = new AudioContext(); + const source = context.createMediaElementSource(this.audio); this.visualizer.setAudioContext(context, source); source.connect(context.destination); diff --git a/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js b/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js index 360a7af6a..e8a36a923 100644 --- a/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js +++ b/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js @@ -315,7 +315,7 @@ class EmojiPickerDropdown extends React.PureComponent { this.setState({ loading: false }); }).catch(() => { - this.setState({ loading: false }); + this.setState({ loading: false, active: false }); }); } diff --git a/app/javascript/mastodon/features/status/components/action_bar.js b/app/javascript/mastodon/features/status/components/action_bar.js index 1c5d5ca0c..d0f1c57d0 100644 --- a/app/javascript/mastodon/features/status/components/action_bar.js +++ b/app/javascript/mastodon/features/status/components/action_bar.js @@ -6,6 +6,7 @@ import ImmutablePropTypes from 'react-immutable-proptypes'; import DropdownMenuContainer from '../../../containers/dropdown_menu_container'; import { defineMessages, injectIntl } from 'react-intl'; import { me, isStaff } from '../../../initial_state'; +import classNames from 'classnames'; const messages = defineMessages({ delete: { id: 'status.delete', defaultMessage: 'Delete' }, @@ -273,7 +274,7 @@ class ActionBar extends React.PureComponent { return (
-
+
{shareButton}
diff --git a/app/javascript/styles/mastodon/boost.scss b/app/javascript/styles/mastodon/boost.scss index 3489428f8..0be3533bc 100644 --- a/app/javascript/styles/mastodon/boost.scss +++ b/app/javascript/styles/mastodon/boost.scss @@ -6,7 +6,7 @@ button.icon-button i.fa-retweet { } } -.status-private button.icon-button i.fa-retweet { +button.icon-button.reblogPrivate i.fa-retweet { background-image: url("data:image/svg+xml;utf8,"); &:hover { diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index ab946470b..a379a7ef4 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -168,11 +168,13 @@ class ActivityPub::Activity end def signed_fetch_account + return Account.find(@options[:delivered_to_account_id]) if @options[:delivered_to_account_id].present? + first_mentioned_local_account || first_local_follower end def first_mentioned_local_account - audience = (as_array(@json['to']) + as_array(@json['cc'])).uniq + audience = (as_array(@json['to']) + as_array(@json['cc'])).map { |x| value_or_id(x) }.uniq local_usernames = audience.select { |uri| ActivityPub::TagManager.instance.local_uri?(uri) } .map { |uri| ActivityPub::TagManager.instance.uri_to_local_id(uri, :username) } diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index 9e108985a..349e8f77e 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -34,12 +34,20 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity private + def audience_to + as_array(@json['to']).map { |x| value_or_id(x) } + end + + def audience_cc + as_array(@json['cc']).map { |x| value_or_id(x) } + end + def visibility_from_audience - if equals_or_includes?(@json['to'], ActivityPub::TagManager::COLLECTIONS[:public]) + if audience_to.include?(ActivityPub::TagManager::COLLECTIONS[:public]) :public - elsif equals_or_includes?(@json['cc'], ActivityPub::TagManager::COLLECTIONS[:public]) + elsif audience_cc.include?(ActivityPub::TagManager::COLLECTIONS[:public]) :unlisted - elsif equals_or_includes?(@json['to'], @account.followers_url) + elsif audience_to.include?(@account.followers_url) :private else :direct diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 08dd98e94..a60b79d15 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -65,11 +65,11 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def audience_to - @object['to'] || @json['to'] + as_array(@object['to'] || @json['to']).map { |x| value_or_id(x) } end def audience_cc - @object['cc'] || @json['cc'] + as_array(@object['cc'] || @json['cc']).map { |x| value_or_id(x) } end def process_status @@ -122,7 +122,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def process_audience - (as_array(audience_to) + as_array(audience_cc)).uniq.each do |audience| + (audience_to + audience_cc).uniq.each do |audience| next if audience == ActivityPub::TagManager::COLLECTIONS[:public] # Unlike with tags, there is no point in resolving accounts we don't already @@ -352,11 +352,11 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def visibility_from_audience - if equals_or_includes?(audience_to, ActivityPub::TagManager::COLLECTIONS[:public]) + if audience_to.include?(ActivityPub::TagManager::COLLECTIONS[:public]) :public - elsif equals_or_includes?(audience_cc, ActivityPub::TagManager::COLLECTIONS[:public]) + elsif audience_cc.include?(ActivityPub::TagManager::COLLECTIONS[:public]) :unlisted - elsif equals_or_includes?(audience_to, @account.followers_url) + elsif audience_to.include?(@account.followers_url) :private else :direct @@ -365,7 +365,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def audience_includes?(account) uri = ActivityPub::TagManager.instance.uri_for(account) - equals_or_includes?(audience_to, uri) || equals_or_includes?(audience_cc, uri) + audience_to.include?(uri) || audience_cc.include?(uri) end def replied_to_status @@ -477,7 +477,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def addresses_local_accounts? return true if @options[:delivered_to_account_id] - local_usernames = (as_array(audience_to) + as_array(audience_cc)).uniq.select { |uri| ActivityPub::TagManager.instance.local_uri?(uri) }.map { |uri| ActivityPub::TagManager.instance.uri_to_local_id(uri, :username) } + local_usernames = (audience_to + audience_cc).uniq.select { |uri| ActivityPub::TagManager.instance.local_uri?(uri) }.map { |uri| ActivityPub::TagManager.instance.uri_to_local_id(uri, :username) } return false if local_usernames.empty? diff --git a/app/lib/activitypub/activity/move.rb b/app/lib/activitypub/activity/move.rb index 2103f503f..7e073f64d 100644 --- a/app/lib/activitypub/activity/move.rb +++ b/app/lib/activitypub/activity/move.rb @@ -20,6 +20,9 @@ class ActivityPub::Activity::Move < ActivityPub::Activity # Initiate a re-follow for each follower MoveWorker.perform_async(origin_account.id, target_account.id) + rescue + unmark_as_processing! + raise end private diff --git a/app/lib/activitypub/activity/reject.rb b/app/lib/activitypub/activity/reject.rb index 8d771ed81..886dddb23 100644 --- a/app/lib/activitypub/activity/reject.rb +++ b/app/lib/activitypub/activity/reject.rb @@ -4,7 +4,7 @@ class ActivityPub::Activity::Reject < ActivityPub::Activity def perform return reject_follow_for_relay if relay_follow? return follow_request_from_object.reject! unless follow_request_from_object.nil? - return UnfollowService.new.call(follow_from_object.target_account, @account) unless follow_from_object.nil? + return UnfollowService.new.call(follow_from_object.account, @account) unless follow_from_object.nil? case @object['type'] when 'Follow' diff --git a/app/lib/webfinger.rb b/app/lib/webfinger.rb new file mode 100644 index 000000000..b2374c494 --- /dev/null +++ b/app/lib/webfinger.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +class Webfinger + class Error < StandardError; end + + class Response + def initialize(body) + @json = Oj.load(body, mode: :strict) + end + + def subject + @json['subject'] + end + + def link(rel, attribute) + links.dig(rel, attribute) + end + + private + + def links + @links ||= @json['links'].map { |link| [link['rel'], link] }.to_h + end + end + + def initialize(uri) + _, @domain = uri.split('@') + + raise ArgumentError, 'Webfinger requested for local account' if @domain.nil? + + @uri = uri + end + + def perform + Response.new(body_from_webfinger) + rescue Oj::ParseError + raise Webfinger::Error, "Invalid JSON in response for #{@uri}" + rescue Addressable::URI::InvalidURIError + raise Webfinger::Error, "Invalid URI for #{@uri}" + end + + private + + def body_from_webfinger(url = standard_url, use_fallback = true) + webfinger_request(url).perform do |res| + if res.code == 200 + res.body_with_limit + elsif res.code == 404 && use_fallback + body_from_host_meta + else + raise Webfinger::Error, "Request for #{@uri} returned HTTP #{res.code}" + end + end + end + + def body_from_host_meta + host_meta_request.perform do |res| + if res.code == 200 + body_from_webfinger(url_from_template(res.body_with_limit), false) + else + raise Webfinger::Error, "Request for #{@uri} returned HTTP #{res.code}" + end + end + end + + def url_from_template(str) + link = Nokogiri::XML(str).at_xpath('//xmlns:Link[@rel="lrdd"]') + + if link.present? + link['template'].gsub('{uri}', @uri) + else + raise Webfinger::Error, "Request for #{@uri} returned host-meta without link to Webfinger" + end + rescue Nokogiri::XML::XPath::SyntaxError + raise Webfinger::Error, "Invalid XML encountered in host-meta for #{@uri}" + end + + def host_meta_request + Request.new(:get, host_meta_url).add_headers('Accept' => 'application/xrd+xml, application/xml, text/xml') + end + + def webfinger_request(url) + Request.new(:get, url).add_headers('Accept' => 'application/jrd+json, application/json') + end + + def standard_url + "https://#{@domain}/.well-known/webfinger?resource=#{@uri}" + end + + def host_meta_url + "https://#{@domain}/.well-known/host-meta" + end +end diff --git a/app/models/account.rb b/app/models/account.rb index 6b7ebda9e..9288c7fe8 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -415,7 +415,7 @@ class Account < ApplicationRecord end def inboxes - urls = reorder(nil).where(protocol: :activitypub).pluck(Arel.sql("distinct coalesce(nullif(accounts.shared_inbox_url, ''), accounts.inbox_url)")) + urls = reorder(nil).where(protocol: :activitypub).group(:preferred_inbox_url).pluck(Arel.sql("coalesce(nullif(accounts.shared_inbox_url, ''), accounts.inbox_url) AS preferred_inbox_url")) DeliveryFailureTracker.without_unavailable(urls) end diff --git a/app/models/account_alias.rb b/app/models/account_alias.rb index 792e9e8d4..3d659142a 100644 --- a/app/models/account_alias.rb +++ b/app/models/account_alias.rb @@ -33,7 +33,7 @@ class AccountAlias < ApplicationRecord def set_uri target_account = ResolveAccountService.new.call(acct) self.uri = ActivityPub::TagManager.instance.uri_for(target_account) unless target_account.nil? - rescue Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error + rescue Webfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error # Validation will take care of it end diff --git a/app/models/account_migration.rb b/app/models/account_migration.rb index 681b5b2cd..4fae98ed7 100644 --- a/app/models/account_migration.rb +++ b/app/models/account_migration.rb @@ -54,7 +54,7 @@ class AccountMigration < ApplicationRecord def set_target_account self.target_account = ResolveAccountService.new.call(acct) - rescue Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error + rescue Webfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error # Validation will take care of it end diff --git a/app/models/account_stat.rb b/app/models/account_stat.rb index c84e4217c..e70b54d79 100644 --- a/app/models/account_stat.rb +++ b/app/models/account_stat.rb @@ -21,26 +21,26 @@ class AccountStat < ApplicationRecord def increment_count!(key) update(attributes_for_increment(key)) - rescue ActiveRecord::StaleObjectError + rescue ActiveRecord::StaleObjectError, ActiveRecord::RecordNotUnique begin reload_with_id rescue ActiveRecord::RecordNotFound - # Nothing to do - else - retry + return end + + retry end def decrement_count!(key) - update(key => [public_send(key) - 1, 0].max) - rescue ActiveRecord::StaleObjectError + update(attributes_for_decrement(key)) + rescue ActiveRecord::StaleObjectError, ActiveRecord::RecordNotUnique begin reload_with_id rescue ActiveRecord::RecordNotFound - # Nothing to do - else - retry + return end + + retry end private @@ -51,8 +51,13 @@ class AccountStat < ApplicationRecord attrs end + def attributes_for_decrement(key) + attrs = { key => [public_send(key) - 1, 0].max } + attrs + end + def reload_with_id - self.id = find_by!(account: account).id if new_record? + self.id = self.class.find_by!(account: account).id if new_record? reload end end diff --git a/app/models/admin/account_action.rb b/app/models/admin/account_action.rb index b30a82369..9edd152f5 100644 --- a/app/models/admin/account_action.rb +++ b/app/models/admin/account_action.rb @@ -142,7 +142,7 @@ class Admin::AccountAction end def status_ids - @report.status_ids if @report && include_statuses + report.status_ids if report && include_statuses end def reports diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb index 0b285fde9..fc6ab5c90 100644 --- a/app/models/form/account_batch.rb +++ b/app/models/form/account_batch.rb @@ -43,7 +43,7 @@ class Form::AccountBatch end def account_domains - accounts.pluck(Arel.sql('distinct domain')).compact + accounts.group(:domain).pluck(:domain).compact end def accounts diff --git a/app/models/form/redirect.rb b/app/models/form/redirect.rb index a7961f8e8..19ee9faed 100644 --- a/app/models/form/redirect.rb +++ b/app/models/form/redirect.rb @@ -32,7 +32,7 @@ class Form::Redirect def set_target_account @target_account = ResolveAccountService.new.call(acct) - rescue Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error + rescue Webfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error # Validation will take care of it end diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 3d93ec75b..663bb0896 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -338,7 +338,7 @@ class MediaAttachment < ApplicationRecord raise Mastodon::StreamValidationError, 'Video has no video stream' if movie.width.nil? || movie.frame_rate.nil? raise Mastodon::DimensionsValidationError, "#{movie.width}x#{movie.height} videos are not supported" if movie.width * movie.height > MAX_VIDEO_MATRIX_LIMIT - raise Mastodon::DimensionsValidationError, "#{movie.frame_rate.to_i}fps videos are not supported" if movie.frame_rate > MAX_VIDEO_FRAME_RATE + raise Mastodon::DimensionsValidationError, "#{movie.frame_rate.floor}fps videos are not supported" if movie.frame_rate.floor > MAX_VIDEO_FRAME_RATE end def set_meta diff --git a/app/models/remote_follow.rb b/app/models/remote_follow.rb index 30b84f7d5..911c06713 100644 --- a/app/models/remote_follow.rb +++ b/app/models/remote_follow.rb @@ -56,7 +56,7 @@ class RemoteFollow if domain.nil? @addressable_template = Addressable::Template.new("#{authorize_interaction_url}?uri={uri}") - elsif redirect_url_link.nil? || redirect_url_link.template.nil? + elsif redirect_uri_template.nil? missing_resource_error else @addressable_template = Addressable::Template.new(redirect_uri_template) @@ -64,16 +64,12 @@ class RemoteFollow end def redirect_uri_template - redirect_url_link.template - end - - def redirect_url_link - acct_resource&.link('http://ostatus.org/schema/1.0/subscribe') + acct_resource&.link('http://ostatus.org/schema/1.0/subscribe', 'template') end def acct_resource @acct_resource ||= webfinger!("acct:#{acct}") - rescue Goldfinger::Error, HTTP::ConnectionError + rescue Webfinger::Error, HTTP::ConnectionError nil end diff --git a/app/models/tag.rb b/app/models/tag.rb index bce76fc16..0870afc41 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -126,7 +126,7 @@ class Tag < ApplicationRecord end def search_for(term, limit = 5, offset = 0, options = {}) - normalized_term = normalize(term.strip).mb_chars.downcase.to_s + normalized_term = normalize(term.strip) pattern = sanitize_sql_like(normalized_term) + '%' query = Tag.listable.where(arel_table[:name].lower.matches(pattern)) query = query.where(arel_table[:name].lower.eq(normalized_term).or(arel_table[:reviewed_at].not_eq(nil))) if options[:exclude_unreviewed] diff --git a/app/models/user.rb b/app/models/user.rb index 306e2d435..0efeed9f7 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -61,7 +61,7 @@ class User < ApplicationRecord devise :two_factor_backupable, otp_number_of_backup_codes: 10 - devise :registerable, :recoverable, :rememberable, :trackable, :validatable, + devise :registerable, :recoverable, :rememberable, :validatable, :confirmable include Omniauthable @@ -161,6 +161,24 @@ class User < ApplicationRecord prepare_new_user! if new_user && approved? end + def update_sign_in!(request, new_sign_in: false) + old_current, new_current = current_sign_in_at, Time.now.utc + self.last_sign_in_at = old_current || new_current + self.current_sign_in_at = new_current + + old_current, new_current = current_sign_in_ip, request.remote_ip + self.last_sign_in_ip = old_current || new_current + self.current_sign_in_ip = new_current + + if new_sign_in + self.sign_in_count ||= 0 + self.sign_in_count += 1 + end + + save(validate: false) unless new_record? + prepare_returning_user! + end + def pending? !approved? end @@ -192,11 +210,6 @@ class User < ApplicationRecord prepare_new_user! end - def update_tracked_fields!(request) - super - prepare_returning_user! - end - def disable_two_factor! self.otp_required_for_login = false otp_backup_codes&.clear diff --git a/app/services/activitypub/fetch_remote_account_service.rb b/app/services/activitypub/fetch_remote_account_service.rb index 83fbf6d07..e5bd0c47c 100644 --- a/app/services/activitypub/fetch_remote_account_service.rb +++ b/app/services/activitypub/fetch_remote_account_service.rb @@ -39,17 +39,16 @@ class ActivityPub::FetchRemoteAccountService < BaseService webfinger = webfinger!("acct:#{@username}@#{@domain}") confirmed_username, confirmed_domain = split_acct(webfinger.subject) - return webfinger.link('self')&.href == @uri if @username.casecmp(confirmed_username).zero? && @domain.casecmp(confirmed_domain).zero? + return webfinger.link('self', 'href') == @uri if @username.casecmp(confirmed_username).zero? && @domain.casecmp(confirmed_domain).zero? webfinger = webfinger!("acct:#{confirmed_username}@#{confirmed_domain}") @username, @domain = split_acct(webfinger.subject) - self_reference = webfinger.link('self') return false unless @username.casecmp(confirmed_username).zero? && @domain.casecmp(confirmed_domain).zero? - return false if self_reference&.href != @uri + return false if webfinger.link('self', 'href') != @uri true - rescue Goldfinger::Error + rescue Webfinger::Error false end diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index 85b915ec6..9f95f1950 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -196,7 +196,7 @@ class ActivityPub::ProcessAccountService < BaseService total_items = collection.is_a?(Hash) && collection['totalItems'].present? && collection['totalItems'].is_a?(Numeric) ? collection['totalItems'] : nil has_first_page = collection.is_a?(Hash) && collection['first'].present? @collections[type] = [total_items, has_first_page] - rescue HTTP::Error, OpenSSL::SSL::SSLError + rescue HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::LengthValidationError @collections[type] = [nil, nil] end diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb index 276eac0c1..e05d02cef 100644 --- a/app/services/fan_out_on_write_service.rb +++ b/app/services/fan_out_on_write_service.rb @@ -6,8 +6,6 @@ class FanOutOnWriteService < BaseService def call(status) raise Mastodon::RaceConditionError if status.visibility.nil? - render_anonymous_payload(status) - if status.direct_visibility? deliver_to_own_conversation(status) elsif status.limited_visibility? @@ -20,6 +18,8 @@ class FanOutOnWriteService < BaseService return if status.account.silenced? || !status.public_visibility? || status.reblog? + render_anonymous_payload(status) + deliver_to_hashtags(status) return if status.reply? && status.in_reply_to_account_id != status.account_id @@ -58,8 +58,10 @@ class FanOutOnWriteService < BaseService def deliver_to_mentioned_followers(status) Rails.logger.debug "Delivering status #{status.id} to limited followers" - FeedInsertWorker.push_bulk(status.mentions.includes(:account).map(&:account).select { |mentioned_account| mentioned_account.local? && mentioned_account.following?(status.account) }) do |follower| - [status.id, follower.id, :home] + status.mentions.joins(:account).merge(status.account.followers_for_local_distribution).select(:id, :account_id).reorder(nil).find_in_batches do |mentions| + FeedInsertWorker.push_bulk(mentions) do |mention| + [status.id, mention.account_id, :home] + end end end diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb index 79af3fc54..8e260811d 100644 --- a/app/services/process_mentions_service.rb +++ b/app/services/process_mentions_service.rb @@ -29,7 +29,7 @@ class ProcessMentionsService < BaseService if mention_undeliverable?(mentioned_account) begin mentioned_account = resolve_account_service.call(Regexp.last_match(1)) - rescue Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::UnexpectedResponseError + rescue Webfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::UnexpectedResponseError mentioned_account = nil end end diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb index ba77552c6..f8d61611f 100644 --- a/app/services/resolve_account_service.rb +++ b/app/services/resolve_account_service.rb @@ -26,12 +26,12 @@ class ResolveAccountService < BaseService @account ||= Account.find_remote(@username, @domain) - return @account if @account&.local? || !webfinger_update_due? + return @account if @account&.local? || @domain.nil? || !webfinger_update_due? # At this point we are in need of a Webfinger query, which may # yield us a different username/domain through a redirect - process_webfinger!(@uri) + @domain = nil if TagManager.instance.local_domain?(@domain) # Because the username/domain pair may be different than what # we already checked, we need to check if we've already got @@ -46,8 +46,8 @@ class ResolveAccountService < BaseService # Now it is certain, it is definitely a remote account, and it # either needs to be created, or updated from fresh data - process_account! - rescue Goldfinger::Error, WebfingerRedirectError, Oj::ParseError => e + fetch_account! + rescue Webfinger::Error, WebfingerRedirectError, Oj::ParseError => e Rails.logger.debug "Webfinger query for #{@uri} failed: #{e}" nil end @@ -76,33 +76,35 @@ class ResolveAccountService < BaseService @uri = [@username, @domain].compact.join('@') end - def process_webfinger!(uri, redirected = false) + def process_webfinger!(uri) @webfinger = webfinger!("acct:#{uri}") - confirmed_username, confirmed_domain = @webfinger.subject.gsub(/\Aacct:/, '').split('@') + confirmed_username, confirmed_domain = split_acct(@webfinger.subject) if confirmed_username.casecmp(@username).zero? && confirmed_domain.casecmp(@domain).zero? @username = confirmed_username @domain = confirmed_domain - @uri = uri - elsif !redirected - return process_webfinger!("#{confirmed_username}@#{confirmed_domain}", true) - else - raise WebfingerRedirectError, "The URI #{uri} tries to hijack #{@username}@#{@domain}" + return end - @domain = nil if TagManager.instance.local_domain?(@domain) + # Account doesn't match, so it may have been redirected + @webfinger = webfinger!("acct:#{confirmed_username}@#{confirmed_domain}") + @username, @domain = split_acct(@webfinger.subject) + + unless confirmed_username.casecmp(@username).zero? && confirmed_domain.casecmp(@domain).zero? + raise WebfingerRedirectError, "The URI #{uri} tries to hijack #{@username}@#{@domain}" + end end - def process_account! + def split_acct(acct) + acct.gsub(/\Aacct:/, '').split('@') + end + + def fetch_account! return unless activitypub_ready? RedisLock.acquire(lock_options) do |lock| if lock.acquired? - @account = Account.find_remote(@username, @domain) - - next if actor_json.nil? - - @account = ActivityPub::ProcessAccountService.new.call(@username, @domain, actor_json) + @account = ActivityPub::FetchRemoteAccountService.new.call(actor_url) else raise Mastodon::RaceConditionError end @@ -118,11 +120,11 @@ class ResolveAccountService < BaseService end def activitypub_ready? - !@webfinger.link('self').nil? && ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@webfinger.link('self').type) + ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@webfinger.link('self', 'type')) end def actor_url - @actor_url ||= @webfinger.link('self').href + @actor_url ||= @webfinger.link('self', 'href') end def actor_json diff --git a/app/views/statuses/_detailed_status.html.haml b/app/views/statuses/_detailed_status.html.haml index 85b2ceea4..b3e9c44fc 100644 --- a/app/views/statuses/_detailed_status.html.haml +++ b/app/views/statuses/_detailed_status.html.haml @@ -29,11 +29,11 @@ - if !status.media_attachments.empty? - if status.media_attachments.first.video? - video = status.media_attachments.first - = react_component :video, src: video.file.url(:original), preview: video.thumbnail.present? ? video.thumbnail.url : video.file.url(:small), blurhash: video.blurhash, sensitive: status.sensitive?, width: 670, height: 380, detailed: true, inline: true, alt: video.description do + = react_component :video, src: full_asset_url(video.file.url(:original)), preview: full_asset_url(video.thumbnail.present? ? video.thumbnail.url : video.file.url(:small)), blurhash: video.blurhash, sensitive: status.sensitive?, width: 670, height: 380, detailed: true, inline: true, alt: video.description do = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } - elsif status.media_attachments.first.audio? - audio = status.media_attachments.first - = react_component :audio, src: audio.file.url(:original), poster: audio.thumbnail.present? ? audio.thumbnail.url : status.account.avatar_static_url, backgroundColor: audio.file.meta.dig('colors', 'background'), foregroundColor: audio.file.meta.dig('colors', 'foreground'), accentColor: audio.file.meta.dig('colors', 'accent'), width: 670, height: 380, alt: audio.description, duration: audio.file.meta.dig('original', 'duration') do + = react_component :audio, src: full_asset_url(audio.file.url(:original)), poster: full_asset_url(audio.thumbnail.present? ? audio.thumbnail.url : status.account.avatar_static_url), backgroundColor: audio.file.meta.dig('colors', 'background'), foregroundColor: audio.file.meta.dig('colors', 'foreground'), accentColor: audio.file.meta.dig('colors', 'accent'), width: 670, height: 380, alt: audio.description, duration: audio.file.meta.dig('original', 'duration') do = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } - else = react_component :media_gallery, height: 380, sensitive: status.sensitive?, standalone: true, autoplay: autoplay, media: status.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } do diff --git a/app/views/statuses/_simple_status.html.haml b/app/views/statuses/_simple_status.html.haml index 67c6c0fd0..363757945 100644 --- a/app/views/statuses/_simple_status.html.haml +++ b/app/views/statuses/_simple_status.html.haml @@ -35,11 +35,11 @@ - if !status.media_attachments.empty? - if status.media_attachments.first.video? - video = status.media_attachments.first - = react_component :video, src: video.file.url(:original), preview: video.thumbnail.present? ? video.thumbnail.url : video.file.url(:small), blurhash: video.blurhash, sensitive: status.sensitive?, width: 610, height: 343, inline: true, alt: video.description do + = react_component :video, src: full_asset_url(video.file.url(:original)), preview: full_asset_url(video.thumbnail.present? ? video.thumbnail.url : video.file.url(:small)), blurhash: video.blurhash, sensitive: status.sensitive?, width: 610, height: 343, inline: true, alt: video.description do = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } - elsif status.media_attachments.first.audio? - audio = status.media_attachments.first - = react_component :audio, src: audio.file.url(:original), poster: audio.thumbnail.present? ? audio.thumbnail.url : status.account.avatar_static_url, backgroundColor: audio.file.meta.dig('colors', 'background'), foregroundColor: audio.file.meta.dig('colors', 'foreground'), accentColor: audio.file.meta.dig('colors', 'accent'), width: 610, height: 343, alt: audio.description, duration: audio.file.meta.dig('original', 'duration') do + = react_component :audio, src: full_asset_url(audio.file.url(:original)), poster: full_asset_url(audio.thumbnail.present? ? audio.thumbnail.url : status.account.avatar_static_url), backgroundColor: audio.file.meta.dig('colors', 'background'), foregroundColor: audio.file.meta.dig('colors', 'foreground'), accentColor: audio.file.meta.dig('colors', 'accent'), width: 610, height: 343, alt: audio.description, duration: audio.file.meta.dig('original', 'duration') do = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } - else = react_component :media_gallery, height: 343, sensitive: status.sensitive?, autoplay: autoplay, media: status.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } do diff --git a/app/views/well_known/host_meta/show.xml.ruby b/app/views/well_known/host_meta/show.xml.ruby index 0a6bdc322..b4e867c5f 100644 --- a/app/views/well_known/host_meta/show.xml.ruby +++ b/app/views/well_known/host_meta/show.xml.ruby @@ -5,7 +5,6 @@ doc << Ox::Element.new('XRD').tap do |xrd| xrd << Ox::Element.new('Link').tap do |link| link['rel'] = 'lrdd' - link['type'] = 'application/xrd+xml' link['template'] = @webfinger_template end end diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 783569451..4144ba8b1 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -15,12 +15,12 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.1.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: 3.1.5 +appVersion: 3.2.2 dependencies: - name: elasticsearch diff --git a/chart/values.yaml.template b/chart/values.yaml.template index 694bc4d42..e1984068c 100644 --- a/chart/values.yaml.template +++ b/chart/values.yaml.template @@ -4,7 +4,7 @@ image: repository: tootsuite/mastodon pullPolicy: Always # https://hub.docker.com/r/tootsuite/mastodon/tags - tag: v3.1.5 + tag: v3.2.2 # alternatively, use `latest` for the latest release or `edge` for the image # built from the most recent commit # diff --git a/db/migrate/20200620164023_add_fixed_lowercase_index_to_accounts.rb b/db/migrate/20200620164023_add_fixed_lowercase_index_to_accounts.rb index c5688681f..c3aa8e33c 100644 --- a/db/migrate/20200620164023_add_fixed_lowercase_index_to_accounts.rb +++ b/db/migrate/20200620164023_add_fixed_lowercase_index_to_accounts.rb @@ -1,10 +1,30 @@ class AddFixedLowercaseIndexToAccounts < ActiveRecord::Migration[5.2] disable_ddl_transaction! + class CorruptionError < StandardError + def cause + nil + end + + def backtrace + [] + end + end + def up - rename_index :accounts, 'index_accounts_on_username_and_domain_lower', 'old_index_accounts_on_username_and_domain_lower' unless index_name_exists?(:accounts, 'old_index_accounts_on_username_and_domain_lower') - add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true, algorithm: :concurrently - remove_index :accounts, name: 'old_index_accounts_on_username_and_domain_lower' + if index_name_exists?(:accounts, 'old_index_accounts_on_username_and_domain_lower') && index_name_exists?(:accounts, 'index_accounts_on_username_and_domain_lower') + remove_index :accounts, name: 'index_accounts_on_username_and_domain_lower' + elsif index_name_exists?(:accounts, 'index_accounts_on_username_and_domain_lower') + rename_index :accounts, 'index_accounts_on_username_and_domain_lower', 'old_index_accounts_on_username_and_domain_lower' + end + + begin + add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true, algorithm: :concurrently + rescue ActiveRecord::RecordNotUnique + raise CorruptionError, 'Migration failed because of index corruption, see https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/#fixing' + end + + remove_index :accounts, name: 'old_index_accounts_on_username_and_domain_lower' if index_name_exists?(:accounts, 'old_index_accounts_on_username_and_domain_lower') end def down diff --git a/lib/cli.rb b/lib/cli.rb index 9162144cc..38df8abcc 100644 --- a/lib/cli.rb +++ b/lib/cli.rb @@ -13,6 +13,7 @@ require_relative 'mastodon/preview_cards_cli' require_relative 'mastodon/cache_cli' require_relative 'mastodon/upgrade_cli' require_relative 'mastodon/email_domain_blocks_cli' +require_relative 'mastodon/maintenance_cli' require_relative 'mastodon/version' module Mastodon @@ -57,6 +58,9 @@ module Mastodon desc 'email_domain_blocks SUBCOMMAND ...ARGS', 'Manage e-mail domain blocks' subcommand 'email_domain_blocks', Mastodon::EmailDomainBlocksCLI + desc 'maintenance SUBCOMMAND ...ARGS', 'Various maintenance utilities' + subcommand 'maintenance', Mastodon::MaintenanceCLI + option :dry_run, type: :boolean desc 'self-destruct', 'Erase the server from the federation' long_desc <<~LONG_DESC diff --git a/lib/mastodon/maintenance_cli.rb b/lib/mastodon/maintenance_cli.rb new file mode 100644 index 000000000..822051ceb --- /dev/null +++ b/lib/mastodon/maintenance_cli.rb @@ -0,0 +1,618 @@ +# frozen_string_literal: true + +require 'tty-prompt' +require_relative '../../config/boot' +require_relative '../../config/environment' +require_relative 'cli_helper' + +module Mastodon + class MaintenanceCLI < Thor + include CLIHelper + + def self.exit_on_failure? + true + end + + MIN_SUPPORTED_VERSION = 2019_10_01_213028 + MAX_SUPPORTED_VERSION = 2020_12_18_054746 + + # Stubs to enjoy ActiveRecord queries while not depending on a particular + # version of the code/database + + class Status < ApplicationRecord; end + class StatusPin < ApplicationRecord; end + class Poll < ApplicationRecord; end + class Report < ApplicationRecord; end + class Tombstone < ApplicationRecord; end + class Favourite < ApplicationRecord; end + class Follow < ApplicationRecord; end + class FollowRequest < ApplicationRecord; end + class Block < ApplicationRecord; end + class Mute < ApplicationRecord; end + class AccountIdentityProof < ApplicationRecord; end + class AccountModerationNote < ApplicationRecord; end + class AccountPin < ApplicationRecord; end + class ListAccount < ApplicationRecord; end + class PollVote < ApplicationRecord; end + class Mention < ApplicationRecord; end + class AccountDomainBlock < ApplicationRecord; end + class AnnouncementReaction < ApplicationRecord; end + class FeaturedTag < ApplicationRecord; end + class CustomEmoji < ApplicationRecord; end + class CustomEmojiCategory < ApplicationRecord; end + class Bookmark < ApplicationRecord; end + class WebauthnCredential < ApplicationRecord; end + + class PreviewCard < ApplicationRecord + self.inheritance_column = false + end + + class MediaAttachment < ApplicationRecord + self.inheritance_column = nil + end + + class AccountStat < ApplicationRecord + belongs_to :account, inverse_of: :account_stat + end + + # Dummy class, to make migration possible across version changes + class Account < ApplicationRecord + has_one :user, inverse_of: :account + has_one :account_stat, inverse_of: :account + + scope :local, -> { where(domain: nil) } + + def local? + domain.nil? + end + + def acct + local? ? username : "#{username}@#{domain}" + end + + # This is a duplicate of the AccountMerging concern because we need it to + # be independent from code version. + def merge_with!(other_account) + # Since it's the same remote resource, the remote resource likely + # already believes we are following/blocking, so it's safe to + # re-attribute the relationships too. However, during the presence + # of the index bug users could have *also* followed the reference + # account already, therefore mass update will not work and we need + # to check for (and skip past) uniqueness errors + + owned_classes = [ + Status, StatusPin, MediaAttachment, Poll, Report, Tombstone, Favourite, + Follow, FollowRequest, Block, Mute, AccountIdentityProof, + AccountModerationNote, AccountPin, AccountStat, ListAccount, + PollVote, Mention + ] + owned_classes << AccountDeletionRequest if ActiveRecord::Base.connection.table_exists?(:account_deletion_requests) + owned_classes << AccountNote if ActiveRecord::Base.connection.table_exists?(:account_notes) + + owned_classes.each do |klass| + klass.where(account_id: other_account.id).find_each do |record| + begin + record.update_attribute(:account_id, id) + rescue ActiveRecord::RecordNotUnique + next + end + end + end + + target_classes = [Follow, FollowRequest, Block, Mute, AccountModerationNote, AccountPin] + target_classes << AccountNote if ActiveRecord::Base.connection.table_exists?(:account_notes) + + target_classes.each do |klass| + klass.where(target_account_id: other_account.id).find_each do |record| + begin + record.update_attribute(:target_account_id, id) + rescue ActiveRecord::RecordNotUnique + next + end + end + end + end + end + + class User < ApplicationRecord + belongs_to :account, inverse_of: :user + end + + desc 'fix-duplicates', 'Fix duplicates in database and rebuild indexes' + long_desc <<~LONG_DESC + Delete or merge duplicate accounts, statuses, emojis, etc. and rebuild indexes. + + This is useful if your database indexes are corrupted because of issues such as https://wiki.postgresql.org/wiki/Locale_data_changes + + Mastodon has to be stopped to run this task, which will take a long time and may be destructive. + LONG_DESC + def fix_duplicates + @prompt = TTY::Prompt.new + + if ActiveRecord::Migrator.current_version < MIN_SUPPORTED_VERSION + @prompt.warn 'Your version of the database schema is too old and is not supported by this script.' + @prompt.warn 'Please update to at least Mastodon 3.0.0 before running this script.' + exit(1) + elsif ActiveRecord::Migrator.current_version > MAX_SUPPORTED_VERSION + @prompt.warn 'Your version of the database schema is more recent than this script, this may cause unexpected errors.' + exit(1) unless @prompt.yes?('Continue anyway?') + end + + @prompt.warn 'This task will take a long time to run and is potentially destructive.' + @prompt.warn 'Please make sure to stop Mastodon and have a backup.' + exit(1) unless @prompt.yes?('Continue?') + + deduplicate_accounts! + deduplicate_users! + deduplicate_account_domain_blocks! + deduplicate_account_identity_proofs! + deduplicate_announcement_reactions! + deduplicate_conversations! + deduplicate_custom_emojis! + deduplicate_custom_emoji_categories! + deduplicate_domain_allows! + deduplicate_domain_blocks! + deduplicate_unavailable_domains! + deduplicate_email_domain_blocks! + deduplicate_media_attachments! + deduplicate_preview_cards! + deduplicate_statuses! + deduplicate_tags! + deduplicate_webauthn_credentials! + + Rails.cache.clear + + @prompt.say 'Finished!' + end + + private + + def deduplicate_accounts! + remove_index_if_exists!(:accounts, 'index_accounts_on_username_and_domain_lower') + + @prompt.say 'Deduplicating accounts… for local accounts, you will be asked to chose which account to keep unchanged.' + + find_duplicate_accounts.each do |row| + accounts = Account.where(id: row['ids'].split(',')).to_a + + if accounts.first.local? + deduplicate_local_accounts!(accounts) + else + deduplicate_remote_accounts!(accounts) + end + end + + @prompt.say 'Restoring index_accounts_on_username_and_domain_lower…' + if ActiveRecord::Migrator.current_version < 20200620164023 + ActiveRecord::Base.connection.add_index :accounts, 'lower (username), lower(domain)', name: 'index_accounts_on_username_and_domain_lower', unique: true + else + ActiveRecord::Base.connection.add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true + end + end + + def deduplicate_users! + remove_index_if_exists!(:users, 'index_users_on_confirmation_token') + remove_index_if_exists!(:users, 'index_users_on_email') + remove_index_if_exists!(:users, 'index_users_on_remember_token') + remove_index_if_exists!(:users, 'index_users_on_reset_password_token') + + @prompt.say 'Deduplicating user records…' + + # Deduplicating email + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users GROUP BY email HAVING count(*) > 1").each do |row| + users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse + ref_user = users.shift + @prompt.warn "Multiple users registered with e-mail address #{ref_user.email}." + @prompt.warn "e-mail will be disabled for the following accounts: #{user.map(&:account).map(&:acct).join(', ')}" + @prompt.warn 'Please reach out to them and set another address with `tootctl account modify` or delete them.' + + i = 0 + users.each do |user| + user.update!(email: "#{i} " + user.email) + end + end + + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE confirmation_token IS NOT NULL GROUP BY confirmation_token HAVING count(*) > 1").each do |row| + users = User.where(id: row['ids'].split(',')).sort_by(&:created_at).reverse.drop(1) + @prompt.warn "Unsetting confirmation token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}" + + users.each do |user| + user.update!(confirmation_token: nil) + end + end + + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1").each do |row| + users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1) + @prompt.warn "Unsetting remember token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}" + + users.each do |user| + user.update!(remember_token: nil) + end + end + + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE reset_password_token IS NOT NULL GROUP BY reset_password_token HAVING count(*) > 1").each do |row| + users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1) + @prompt.warn "Unsetting password reset token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}" + + users.each do |user| + user.update!(reset_password_token: nil) + end + end + + @prompt.say 'Restoring users indexes…' + ActiveRecord::Base.connection.add_index :users, ['confirmation_token'], name: 'index_users_on_confirmation_token', unique: true + ActiveRecord::Base.connection.add_index :users, ['email'], name: 'index_users_on_email', unique: true + ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true + ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true + end + + def deduplicate_account_domain_blocks! + remove_index_if_exists!(:account_domain_blocks, 'index_account_domain_blocks_on_account_id_and_domain') + + @prompt.say 'Removing duplicate account domain blocks…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_domain_blocks GROUP BY account_id, domain HAVING count(*) > 1").each do |row| + AccountDomainBlock.where(id: row['ids'].split(',').drop(1)).delete_all + end + + @prompt.say 'Restoring account domain blocks indexes…' + ActiveRecord::Base.connection.add_index :account_domain_blocks, ['account_id', 'domain'], name: 'index_account_domain_blocks_on_account_id_and_domain', unique: true + end + + def deduplicate_account_identity_proofs! + remove_index_if_exists!(:account_identity_proofs, 'index_account_proofs_on_account_and_provider_and_username') + + @prompt.say 'Removing duplicate account identity proofs…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_identity_proofs GROUP BY account_id, provider, provider_username HAVING count(*) > 1").each do |row| + AccountIdentityProof.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring account identity proofs indexes…' + ActiveRecord::Base.connection.add_index :account_identity_proofs, ['account_id', 'provider', 'provider_username'], name: 'index_account_proofs_on_account_and_provider_and_username', unique: true + end + + def deduplicate_announcement_reactions! + return unless ActiveRecord::Base.connection.table_exists?(:announcement_reactions) + + remove_index_if_exists!(:announcement_reactions, 'index_announcement_reactions_on_account_id_and_announcement_id') + + @prompt.say 'Removing duplicate account identity proofs…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM announcement_reactions GROUP BY account_id, announcement_id, name HAVING count(*) > 1").each do |row| + AnnouncementReaction.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring announcement_reactions indexes…' + ActiveRecord::Base.connection.add_index :announcement_reactions, ['account_id', 'announcement_id', 'name'], name: 'index_announcement_reactions_on_account_id_and_announcement_id', unique: true + end + + def deduplicate_conversations! + remove_index_if_exists!(:conversations, 'index_conversations_on_uri') + + @prompt.say 'Deduplicating conversations…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM conversations WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row| + conversations = Conversation.where(id: row['ids'].split(',')).sort_by(&:id).reverse + + ref_conversation = conversations.shift + + conversations.each do |other| + merge_conversations!(ref_conversation, other) + other.destroy + end + end + + @prompt.say 'Restoring conversations indexes…' + ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true + end + + def deduplicate_custom_emojis! + remove_index_if_exists!(:custom_emojis, 'index_custom_emojis_on_shortcode_and_domain') + + @prompt.say 'Deduplicating custom_emojis…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emojis GROUP BY shortcode, domain HAVING count(*) > 1").each do |row| + emojis = CustomEmoji.where(id: row['ids'].split(',')).sort_by(&:id).reverse + + ref_emoji = emojis.shift + + emojis.each do |other| + merge_custom_emojis!(ref_emoji, other) + other.destroy + end + end + + @prompt.say 'Restoring custom_emojis indexes…' + ActiveRecord::Base.connection.add_index :custom_emojis, ['shortcode', 'domain'], name: 'index_custom_emojis_on_shortcode_and_domain', unique: true + end + + def deduplicate_custom_emoji_categories! + remove_index_if_exists!(:custom_emoji_categories, 'index_custom_emoji_categories_on_name') + + @prompt.say 'Deduplicating custom_emoji_categories…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emoji_categories GROUP BY name HAVING count(*) > 1").each do |row| + categories = CustomEmojiCategory.where(id: row['ids'].split(',')).sort_by(&:id).reverse + + ref_category = categories.shift + + categories.each do |other| + merge_custom_emoji_categories!(ref_category, other) + other.destroy + end + end + + @prompt.say 'Restoring custom_emoji_categories indexes…' + ActiveRecord::Base.connection.add_index :custom_emoji_categories, ['name'], name: 'index_custom_emoji_categories_on_name', unique: true + end + + def deduplicate_domain_allows! + remove_index_if_exists!(:domain_allows, 'index_domain_allows_on_domain') + + @prompt.say 'Deduplicating domain_allows…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_allows GROUP BY domain HAVING count(*) > 1").each do |row| + DomainAllow.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring domain_allows indexes…' + ActiveRecord::Base.connection.add_index :domain_allows, ['domain'], name: 'index_domain_allows_on_domain', unique: true + end + + def deduplicate_domain_blocks! + remove_index_if_exists!(:domain_blocks, 'index_domain_blocks_on_domain') + + @prompt.say 'Deduplicating domain_allows…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row| + domain_blocks = DomainBlock.where(id: row['ids'].split(',')).by_severity.reverse.to_a + + reject_media = domain_blocks.any?(&:reject_media?) + reject_reports = domain_blocks.any?(&:reject_reports?) + + reference_block = domain_blocks.shift + + private_comment = domain_blocks.reduce(reference_block.private_comment.presence) { |a, b| a || b.private_comment.presence } + public_comment = domain_blocks.reduce(reference_block.public_comment.presence) { |a, b| a || b.public_comment.presence } + + reference_block.update!(reject_media: reject_media, reject_reports: reject_reports, private_comment: private_comment, public_comment: public_comment) + + domain_blocks.each(&:destroy) + end + + @prompt.say 'Restoring domain_blocks indexes…' + ActiveRecord::Base.connection.add_index :domain_blocks, ['domain'], name: 'index_domain_blocks_on_domain', unique: true + end + + def deduplicate_unavailable_domains! + return unless ActiveRecord::Base.connection.table_exists?(:unavailable_domains) + + remove_index_if_exists!(:unavailable_domains, 'index_unavailable_domains_on_domain') + + @prompt.say 'Deduplicating unavailable_domains…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM unavailable_domains GROUP BY domain HAVING count(*) > 1").each do |row| + UnavailableDomain.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring domain_allows indexes…' + ActiveRecord::Base.connection.add_index :unavailable_domains, ['domain'], name: 'index_unavailable_domains_on_domain', unique: true + end + + def deduplicate_email_domain_blocks! + remove_index_if_exists!(:email_domain_blocks, 'index_email_domain_blocks_on_domain') + + @prompt.say 'Deduplicating email_domain_blocks…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM email_domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row| + domain_blocks = EmailDomainBlock.where(id: row['ids'].split(',')).sort_by { |b| b.parent.nil? ? 1 : 0 }.to_a + domain_blocks.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring email_domain_blocks indexes…' + ActiveRecord::Base.connection.add_index :email_domain_blocks, ['domain'], name: 'index_email_domain_blocks_on_domain', unique: true + end + + def deduplicate_media_attachments! + remove_index_if_exists!(:media_attachments, 'index_media_attachments_on_shortcode') + + @prompt.say 'Deduplicating media_attachments…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM media_attachments WHERE shortcode IS NOT NULL GROUP BY shortcode HAVING count(*) > 1").each do |row| + MediaAttachment.where(id: row['ids'].split(',').drop(1)).update_all(shortcode: nil) + end + + @prompt.say 'Restoring media_attachments indexes…' + ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true + end + + def deduplicate_preview_cards! + remove_index_if_exists!(:preview_cards, 'index_preview_cards_on_url') + + @prompt.say 'Deduplicating preview_cards…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM preview_cards GROUP BY url HAVING count(*) > 1").each do |row| + PreviewCard.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring preview_cards indexes…' + ActiveRecord::Base.connection.add_index :preview_cards, ['url'], name: 'index_preview_cards_on_url', unique: true + end + + def deduplicate_statuses! + remove_index_if_exists!(:statuses, 'index_statuses_on_uri') + + @prompt.say 'Deduplicating statuses…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM statuses WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row| + statuses = Status.where(id: row['ids'].split(',')).sort_by(&:id) + ref_status = statuses.shift + statuses.each do |status| + merge_statuses!(ref_status, status) if status.account_id == ref_status.account_id + status.destroy + end + end + + @prompt.say 'Restoring statuses indexes…' + ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true + end + + def deduplicate_tags! + remove_index_if_exists!(:tags, 'index_tags_on_name_lower') + + @prompt.say 'Deduplicating tags…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM tags GROUP BY lower((name)::text) HAVING count(*) > 1").each do |row| + tags = Tag.where(id: row['ids'].split(',')).sort_by { |t| [t.usable?, t.trendable?, t.listable?].count(false) } + ref_tag = tags.shift + tags.each do |tag| + merge_tags!(ref_tag, tag) + tag.destroy + end + end + + @prompt.say 'Restoring tags indexes…' + ActiveRecord::Base.connection.add_index :tags, 'lower((name)::text)', name: 'index_tags_on_name_lower', unique: true + end + + def deduplicate_webauthn_credentials! + return unless ActiveRecord::Base.connection.table_exists?(:webauthn_credentials) + + remove_index_if_exists!(:webauthn_credentials, 'index_webauthn_credentials_on_external_id') + + @prompt.say 'Deduplicating webauthn_credentials…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webauthn_credentials GROUP BY external_id HAVING count(*) > 1").each do |row| + WebauthnCredential.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring webauthn_credentials indexes…' + ActiveRecord::Base.connection.add_index :webauthn_credentials, ['external_id'], name: 'index_webauthn_credentials_on_external_id', unique: true + end + + def deduplicate_local_accounts!(accounts) + accounts = accounts.sort_by(&:id).reverse + + @prompt.warn "Multiple local accounts were found for username '#{accounts.first.username}'." + @prompt.warn 'All those accounts are distinct accounts but only the most recently-created one is fully-functionnal.' + + accounts.each_with_index do |account, idx| + @prompt.say '%2d. %s: created at: %s; updated at: %s; last logged in at: %s; statuses: %5d; last status at: %s' % [idx, account.username, account.created_at, account.updated_at, account.user&.last_sign_in_at&.to_s || 'N/A', account.account_stat&.statuses_count || 0, account.account_stat&.last_status_at || 'N/A'] + end + + @prompt.say 'Please chose the one to keep unchanged, other ones will be automatically renamed.' + + ref_id = @prompt.ask('Account to keep unchanged:') do |q| + q.required true + q.default 0 + q.convert :int + end + + accounts.delete_at(ref_id) + + i = 0 + accounts.each do |account| + i += 1 + username = account.username + "_#{i}" + + while Account.local.exists?(username: username) + i += 1 + username = account.username + "_#{i}" + end + + account.update!(username: username) + end + end + + def deduplicate_remote_accounts!(accounts) + accounts = accounts.sort_by(&:updated_at).reverse + + reference_account = accounts.shift + + accounts.each do |other_account| + if other_account.public_key == reference_account.public_key + # The accounts definitely point to the same resource, so + # it's safe to re-attribute content and relationships + reference_account.merge_with!(other_account) + end + + other_account.destroy + end + end + + def merge_conversations!(main_conv, duplicate_conv) + owned_classes = [ConversationMute, AccountConversation] + owned_classes.each do |klass| + klass.where(conversation_id: duplicate_conv.id).find_each do |record| + begin + record.update_attribute(:account_id, main_conv.id) + rescue ActiveRecord::RecordNotUnique + next + end + end + end + end + + def merge_custom_emojis!(main_emoji, duplicate_emoji) + owned_classes = [AnnouncementReaction] + owned_classes.each do |klass| + klass.where(custom_emoji_id: duplicate_emoji.id).update_all(custom_emoji_id: main_emoji.id) + end + end + + def merge_custom_emoji_categories!(main_category, duplicate_category) + owned_classes = [CustomEmoji] + owned_classes.each do |klass| + klass.where(category_id: duplicate_category.id).update_all(category_id: main_category.id) + end + end + + def merge_statuses!(main_status, duplicate_status) + owned_classes = [Favourite, Mention, Poll] + owned_classes << Bookmark if ActiveRecord::Base.connection.table_exists?(:bookmarks) + owned_classes.each do |klass| + klass.where(status_id: duplicate_status.id).find_each do |record| + begin + record.update_attribute(:status_id, main_status.id) + rescue ActiveRecord::RecordNotUnique + next + end + end + end + + StatusPin.where(account_id: main_status.account_id, status_id: duplicate_status.id).find_each do |record| + begin + record.update_attribute(:status_id, main_status.id) + rescue ActiveRecord::RecordNotUnique + next + end + end + + Status.where(in_reply_to_id: duplicate_status.id).find_each do |record| + begin + record.update_attribute(:in_reply_to_id, main_status.id) + rescue ActiveRecord::RecordNotUnique + next + end + end + + Status.where(reblog_of_id: duplicate_status.id).find_each do |record| + begin + record.update_attribute(:reblog_of_id, main_status.id) + rescue ActiveRecord::RecordNotUnique + next + end + end + end + + def merge_tags!(main_tag, duplicate_tag) + [FeaturedTag].each do |klass| + klass.where(tag_id: duplicate_tag.id).find_each do |record| + begin + record.update_attribute(:tag_id, main_tag.id) + rescue ActiveRecord::RecordNotUnique + next + end + end + end + end + + def find_duplicate_accounts + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM accounts GROUP BY lower(username), COALESCE(lower(domain), '') HAVING count(*) > 1") + end + + def remove_index_if_exists!(table, name) + ActiveRecord::Base.connection.remove_index(table, name: name) + rescue ArgumentError + nil + rescue ActiveRecord::StatementInvalid + nil + end + end +end diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index 2a4e3e379..54da5b2cd 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -31,7 +31,7 @@ module Mastodon processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where('created_at < ?', time_ago)) do |media_attachment| next if media_attachment.file.blank? - size = media_attachment.file_file_size + (media_attachment.thumbnail_file_size || 0) + size = (media_attachment.file_file_size || 0) + (media_attachment.thumbnail_file_size || 0) unless options[:dry_run] media_attachment.file.destroy @@ -89,7 +89,7 @@ module Mastodon path_segments = object.key.split('/') path_segments.delete('cache') - if path_segments.size != 7 + unless [7, 10].include?(path_segments.size) progress.log(pastel.yellow("Unrecognized file found: #{object.key}")) next end @@ -133,7 +133,7 @@ module Mastodon path_segments = key.split(File::SEPARATOR) path_segments.delete('cache') - if path_segments.size != 7 + unless [7, 10].include?(path_segments.size) progress.log(pastel.yellow("Unrecognized file found: #{key}")) next end @@ -258,7 +258,7 @@ module Mastodon path_segments = path.split('/')[2..-1] path_segments.delete('cache') - if path_segments.size != 7 + unless [7, 10].include?(path_segments.size) say('Not a media URL', :red) exit(1) end @@ -311,7 +311,7 @@ module Mastodon segments = object.key.split('/') segments.delete('cache') - next if segments.size != 7 + next unless [7, 10].include?(segments.size) model_name = segments.first.classify record_id = segments[2..-2].join.to_i diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 7aa6cb2c7..89cedbb87 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 0 + 2 end def flags diff --git a/lib/paperclip/color_extractor.rb b/lib/paperclip/color_extractor.rb index 44fe5ff1d..c8bb771a0 100644 --- a/lib/paperclip/color_extractor.rb +++ b/lib/paperclip/color_extractor.rb @@ -5,6 +5,7 @@ require 'mime/types/columnar' module Paperclip class ColorExtractor < Paperclip::Processor MIN_CONTRAST = 3.0 + ACCENT_MIN_CONTRAST = 2.0 FREQUENCY_THRESHOLD = 0.01 def make @@ -26,8 +27,9 @@ module Paperclip foreground_palette.each do |color| distance = ColorDiff.between(background_color, color) + contrast = w3c_contrast(background_color, color) - if distance > max_distance + if distance > max_distance && contrast >= ACCENT_MIN_CONTRAST max_distance = distance max_distance_color = color end @@ -77,8 +79,8 @@ module Paperclip private def w3c_contrast(color1, color2) - luminance1 = (0.2126 * color1.r + 0.7152 * color1.g + 0.0722 * color1.b) + 0.05 - luminance2 = (0.2126 * color2.r + 0.7152 * color2.g + 0.0722 * color2.b) + 0.05 + luminance1 = color1.to_xyz.y * 0.01 + 0.05 + luminance2 = color2.to_xyz.y * 0.01 + 0.05 if luminance1 > luminance2 luminance1 / luminance2 diff --git a/lib/paperclip/response_with_limit_adapter.rb b/lib/paperclip/response_with_limit_adapter.rb index 7d897b8d6..17a2abd25 100644 --- a/lib/paperclip/response_with_limit_adapter.rb +++ b/lib/paperclip/response_with_limit_adapter.rb @@ -16,10 +16,10 @@ module Paperclip private def cache_current_values - @original_filename = filename_from_content_disposition || filename_from_path || 'data' + @original_filename = filename_from_content_disposition.presence || filename_from_path.presence || 'data' @size = @target.response.content_length @tempfile = copy_to_tempfile(@target) - @content_type = @target.response.mime_type || ContentTypeDetector.new(@tempfile.path).detect + @content_type = ContentTypeDetector.new(@tempfile.path).detect end def copy_to_tempfile(source) diff --git a/spec/controllers/accounts_controller_spec.rb b/spec/controllers/accounts_controller_spec.rb index 93bf2c83f..b04f4650b 100644 --- a/spec/controllers/accounts_controller_spec.rb +++ b/spec/controllers/accounts_controller_spec.rb @@ -348,24 +348,8 @@ RSpec.describe AccountsController, type: :controller do context 'in authorized fetch mode' do let(:authorized_fetch_mode) { true } - it 'returns http success' do - expect(response).to have_http_status(200) - end - - it 'returns application/activity+json' do - expect(response.content_type).to eq 'application/activity+json' - end - - it_behaves_like 'cachable response' - - it 'returns Vary header with Signature' do - expect(response.headers['Vary']).to include 'Signature' - end - - it 'renders bare minimum account' do - json = body_as_json - expect(json).to include(:id, :type, :preferredUsername, :inbox, :publicKey) - expect(json).to_not include(:name, :summary) + it 'returns http unauthorized' do + expect(response).to have_http_status(401) end end end diff --git a/spec/controllers/auth/sessions_controller_spec.rb b/spec/controllers/auth/sessions_controller_spec.rb index c387842cd..4560d64a8 100644 --- a/spec/controllers/auth/sessions_controller_spec.rb +++ b/spec/controllers/auth/sessions_controller_spec.rb @@ -215,7 +215,7 @@ RSpec.describe Auth::SessionsController, type: :controller do context 'using a valid OTP' do before do - post :create, params: { user: { otp_attempt: user.current_otp } }, session: { attempt_user_id: user.id } + post :create, params: { user: { otp_attempt: user.current_otp } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s } end it 'redirects to home' do @@ -230,7 +230,7 @@ RSpec.describe Auth::SessionsController, type: :controller do context 'when the server has an decryption error' do before do allow_any_instance_of(User).to receive(:validate_and_consume_otp!).and_raise(OpenSSL::Cipher::CipherError) - post :create, params: { user: { otp_attempt: user.current_otp } }, session: { attempt_user_id: user.id } + post :create, params: { user: { otp_attempt: user.current_otp } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s } end it 'shows a login error' do @@ -244,7 +244,7 @@ RSpec.describe Auth::SessionsController, type: :controller do context 'using a valid recovery code' do before do - post :create, params: { user: { otp_attempt: recovery_codes.first } }, session: { attempt_user_id: user.id } + post :create, params: { user: { otp_attempt: recovery_codes.first } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s } end it 'redirects to home' do @@ -258,7 +258,7 @@ RSpec.describe Auth::SessionsController, type: :controller do context 'using an invalid OTP' do before do - post :create, params: { user: { otp_attempt: 'wrongotp' } }, session: { attempt_user_id: user.id } + post :create, params: { user: { otp_attempt: 'wrongotp' } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s } end it 'shows a login error' do @@ -302,7 +302,7 @@ RSpec.describe Auth::SessionsController, type: :controller do context 'using a valid sign in token' do before do user.generate_sign_in_token && user.save - post :create, params: { user: { sign_in_token_attempt: user.sign_in_token } }, session: { attempt_user_id: user.id } + post :create, params: { user: { sign_in_token_attempt: user.sign_in_token } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s } end it 'redirects to home' do @@ -316,7 +316,7 @@ RSpec.describe Auth::SessionsController, type: :controller do context 'using an invalid sign in token' do before do - post :create, params: { user: { sign_in_token_attempt: 'wrongotp' } }, session: { attempt_user_id: user.id } + post :create, params: { user: { sign_in_token_attempt: 'wrongotp' } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s } end it 'shows a login error' do diff --git a/spec/controllers/remote_follow_controller_spec.rb b/spec/controllers/remote_follow_controller_spec.rb index 3ef8f14d9..7312dde58 100644 --- a/spec/controllers/remote_follow_controller_spec.rb +++ b/spec/controllers/remote_follow_controller_spec.rb @@ -43,8 +43,7 @@ describe RemoteFollowController do end it 'renders new when template is nil' do - link_with_nil_template = double(template: nil) - resource_with_link = double(link: link_with_nil_template) + resource_with_link = double(link: nil) allow_any_instance_of(WebfingerHelper).to receive(:webfinger!).with('acct:user@example.com').and_return(resource_with_link) post :create, params: { account_username: @account.to_param, remote_follow: { acct: 'user@example.com' } } @@ -55,8 +54,7 @@ describe RemoteFollowController do context 'when webfinger values are good' do before do - link_with_template = double(template: 'http://example.com/follow_me?acct={uri}') - resource_with_link = double(link: link_with_template) + resource_with_link = double(link: 'http://example.com/follow_me?acct={uri}') allow_any_instance_of(WebfingerHelper).to receive(:webfinger!).with('acct:user@example.com').and_return(resource_with_link) post :create, params: { account_username: @account.to_param, remote_follow: { acct: 'user@example.com' } } end @@ -78,8 +76,8 @@ describe RemoteFollowController do expect(response).to render_template(:new) end - it 'renders new with error when goldfinger fails' do - allow_any_instance_of(WebfingerHelper).to receive(:webfinger!).with('acct:user@example.com').and_raise(Goldfinger::Error) + it 'renders new with error when webfinger fails' do + allow_any_instance_of(WebfingerHelper).to receive(:webfinger!).with('acct:user@example.com').and_raise(Webfinger::Error) post :create, params: { account_username: @account.to_param, remote_follow: { acct: 'user@example.com' } } expect(response).to render_template(:new) diff --git a/spec/controllers/well_known/host_meta_controller_spec.rb b/spec/controllers/well_known/host_meta_controller_spec.rb index b43ae19d8..643ba9cd3 100644 --- a/spec/controllers/well_known/host_meta_controller_spec.rb +++ b/spec/controllers/well_known/host_meta_controller_spec.rb @@ -12,7 +12,7 @@ describe WellKnown::HostMetaController, type: :controller do expect(response.body).to eq < - + XML end diff --git a/spec/lib/activitypub/activity/announce_spec.rb b/spec/lib/activitypub/activity/announce_spec.rb index 60fd96a18..b93fcbe66 100644 --- a/spec/lib/activitypub/activity/announce_spec.rb +++ b/spec/lib/activitypub/activity/announce_spec.rb @@ -73,6 +73,26 @@ RSpec.describe ActivityPub::Activity::Announce do expect(sender.reblogged?(sender.statuses.first)).to be true end end + + context 'self-boost of a previously unknown status with correct attributedTo, inlined Collection in audience' do + let(:object_json) do + { + id: 'https://example.com/actor#bar', + type: 'Note', + content: 'Lorem ipsum', + attributedTo: 'https://example.com/actor', + to: { + 'type': 'OrderedCollection', + 'id': 'http://example.com/followers', + 'first': 'http://example.com/followers?page=true', + } + } + end + + it 'creates a reblog by sender of status' do + expect(sender.reblogged?(sender.statuses.first)).to be true + end + end end context 'when the status belongs to a local user' do diff --git a/spec/lib/activitypub/activity/create_spec.rb b/spec/lib/activitypub/activity/create_spec.rb index 2ac4acc12..d2e9fe33c 100644 --- a/spec/lib/activitypub/activity/create_spec.rb +++ b/spec/lib/activitypub/activity/create_spec.rb @@ -18,6 +18,7 @@ RSpec.describe ActivityPub::Activity::Create do stub_request(:get, 'http://example.com/attachment.png').to_return(request_fixture('avatar.txt')) stub_request(:get, 'http://example.com/emoji.png').to_return(body: attachment_fixture('emojo.png')) + stub_request(:get, 'http://example.com/emojib.png').to_return(body: attachment_fixture('emojo.png'), headers: { 'Content-Type' => 'application/octet-stream' }) end describe '#perform' do @@ -120,6 +121,28 @@ RSpec.describe ActivityPub::Activity::Create do end end + context 'private with inlined Collection in audience' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + to: { + 'type': 'OrderedCollection', + 'id': 'http://example.com/followers', + 'first': 'http://example.com/followers?page=true', + } + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.visibility).to eq 'private' + end + end + context 'limited' do let(:recipient) { Fabricate(:account) } @@ -451,6 +474,32 @@ RSpec.describe ActivityPub::Activity::Create do end end + context 'with emojis served with invalid content-type' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum :tinkong:', + tag: [ + { + type: 'Emoji', + icon: { + url: 'http://example.com/emojib.png', + }, + name: 'tinkong', + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.emojis.map(&:shortcode)).to include('tinkong') + end + end + context 'with emojis missing name' do let(:object_json) do { diff --git a/spec/lib/activitypub/activity/reject_spec.rb b/spec/lib/activitypub/activity/reject_spec.rb index e7205df8d..fed4cd8cd 100644 --- a/spec/lib/activitypub/activity/reject_spec.rb +++ b/spec/lib/activitypub/activity/reject_spec.rb @@ -3,6 +3,14 @@ require 'rails_helper' RSpec.describe ActivityPub::Activity::Reject do let(:sender) { Fabricate(:account) } let(:recipient) { Fabricate(:account) } + let(:object_json) do + { + id: 'bar', + type: 'Follow', + actor: ActivityPub::TagManager.instance.uri_for(recipient), + object: ActivityPub::TagManager.instance.uri_for(sender), + } + end let(:json) do { @@ -10,29 +18,105 @@ RSpec.describe ActivityPub::Activity::Reject do id: 'foo', type: 'Reject', actor: ActivityPub::TagManager.instance.uri_for(sender), - object: { - id: 'bar', - type: 'Follow', - actor: ActivityPub::TagManager.instance.uri_for(recipient), - object: ActivityPub::TagManager.instance.uri_for(sender), - }, + object: object_json, }.with_indifferent_access end describe '#perform' do subject { described_class.new(json, sender) } - before do - Fabricate(:follow_request, account: recipient, target_account: sender) - subject.perform + context 'rejecting a pending follow request by target' do + before do + Fabricate(:follow_request, account: recipient, target_account: sender) + subject.perform + end + + it 'does not create a follow relationship' do + expect(recipient.following?(sender)).to be false + end + + it 'removes the follow request' do + expect(recipient.requested?(sender)).to be false + end end - it 'does not create a follow relationship' do - expect(recipient.following?(sender)).to be false + context 'rejecting a pending follow request by uri' do + before do + Fabricate(:follow_request, account: recipient, target_account: sender, uri: 'bar') + subject.perform + end + + it 'does not create a follow relationship' do + expect(recipient.following?(sender)).to be false + end + + it 'removes the follow request' do + expect(recipient.requested?(sender)).to be false + end end - it 'removes the follow request' do - expect(recipient.requested?(sender)).to be false + context 'rejecting a pending follow request by uri only' do + let(:object_json) { 'bar' } + + before do + Fabricate(:follow_request, account: recipient, target_account: sender, uri: 'bar') + subject.perform + end + + it 'does not create a follow relationship' do + expect(recipient.following?(sender)).to be false + end + + it 'removes the follow request' do + expect(recipient.requested?(sender)).to be false + end + end + + context 'rejecting an existing follow relationship by target' do + before do + Fabricate(:follow, account: recipient, target_account: sender) + subject.perform + end + + it 'removes the follow relationship' do + expect(recipient.following?(sender)).to be false + end + + it 'does not create a follow request' do + expect(recipient.requested?(sender)).to be false + end + end + + context 'rejecting an existing follow relationship by uri' do + before do + Fabricate(:follow, account: recipient, target_account: sender, uri: 'bar') + subject.perform + end + + it 'removes the follow relationship' do + expect(recipient.following?(sender)).to be false + end + + it 'does not create a follow request' do + expect(recipient.requested?(sender)).to be false + end + end + + context 'rejecting an existing follow relationship by uri only' do + let(:object_json) { 'bar' } + + before do + Fabricate(:follow, account: recipient, target_account: sender, uri: 'bar') + subject.perform + end + + it 'removes the follow relationship' do + expect(recipient.following?(sender)).to be false + end + + it 'does not create a follow request' do + expect(recipient.requested?(sender)).to be false + end end end diff --git a/spec/lib/feed_manager_spec.rb b/spec/lib/feed_manager_spec.rb index 22eddd2ab..2d1f6cf96 100644 --- a/spec/lib/feed_manager_spec.rb +++ b/spec/lib/feed_manager_spec.rb @@ -108,6 +108,7 @@ RSpec.describe FeedManager do it 'returns false for status by followee mentioning another account' do bob.follow!(alice) + jeff.follow!(alice) status = PostStatusService.new.call(alice, text: 'Hey @jeff') expect(FeedManager.instance.filter?(:home, status, bob.id)).to be false end diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb index 98d29e6f3..75f628076 100644 --- a/spec/models/account_spec.rb +++ b/spec/models/account_spec.rb @@ -817,4 +817,27 @@ RSpec.describe Account, type: :model do include_examples 'AccountAvatar', :account include_examples 'AccountHeader', :account + + describe '#increment_count!' do + subject { Fabricate(:account) } + + it 'increments the count in multi-threaded an environment when account_stat is not yet initialized' do + subject + + increment_by = 15 + wait_for_start = true + + threads = Array.new(increment_by) do + Thread.new do + true while wait_for_start + Account.find(subject.id).increment_count!(:followers_count) + end + end + + wait_for_start = false + threads.each(&:join) + + expect(subject.reload.followers_count).to eq 15 + end + end end diff --git a/spec/services/resolve_account_service_spec.rb b/spec/services/resolve_account_service_spec.rb index cea942e39..92c837050 100644 --- a/spec/services/resolve_account_service_spec.rb +++ b/spec/services/resolve_account_service_spec.rb @@ -4,23 +4,76 @@ RSpec.describe ResolveAccountService, type: :service do subject { described_class.new } before do - stub_request(:get, "https://quitter.no/.well-known/host-meta").to_return(request_fixture('.host-meta.txt')) - stub_request(:get, "https://example.com/.well-known/webfinger?resource=acct:catsrgr8@example.com").to_return(status: 404) stub_request(:get, "https://example.com/.well-known/host-meta").to_return(status: 404) stub_request(:get, "https://quitter.no/avatar/7477-300-20160211190340.png").to_return(request_fixture('avatar.txt')) - stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:catsrgr8@quitter.no").to_return(status: 404) stub_request(:get, "https://ap.example.com/.well-known/webfinger?resource=acct:foo@ap.example.com").to_return(request_fixture('activitypub-webfinger.txt')) stub_request(:get, "https://ap.example.com/users/foo").to_return(request_fixture('activitypub-actor.txt')) stub_request(:get, "https://ap.example.com/users/foo.atom").to_return(request_fixture('activitypub-feed.txt')) stub_request(:get, %r{https://ap.example.com/users/foo/\w+}).to_return(status: 404) end - it 'raises error if no such user can be resolved via webfinger' do - expect(subject.call('catsrgr8@quitter.no')).to be_nil + context 'when there is an LRDD endpoint but no resolvable account' do + before do + stub_request(:get, "https://quitter.no/.well-known/host-meta").to_return(request_fixture('.host-meta.txt')) + stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:catsrgr8@quitter.no").to_return(status: 404) + end + + it 'returns nil' do + expect(subject.call('catsrgr8@quitter.no')).to be_nil + end end - it 'raises error if the domain does not have webfinger' do - expect(subject.call('catsrgr8@example.com')).to be_nil + context 'when there is no LRDD endpoint nor resolvable account' do + before do + stub_request(:get, "https://example.com/.well-known/webfinger?resource=acct:catsrgr8@example.com").to_return(status: 404) + end + + it 'returns nil' do + expect(subject.call('catsrgr8@example.com')).to be_nil + end + end + + context 'with a legitimate webfinger redirection' do + before do + webfinger = { subject: 'acct:foo@ap.example.com', links: [{ rel: 'self', href: 'https://ap.example.com/users/foo', type: 'application/activity+json' }] } + stub_request(:get, 'https://redirected.example.com/.well-known/webfinger?resource=acct:Foo@redirected.example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' }) + end + + it 'returns new remote account' do + account = subject.call('Foo@redirected.example.com') + + expect(account.activitypub?).to eq true + expect(account.acct).to eq 'foo@ap.example.com' + expect(account.inbox_url).to eq 'https://ap.example.com/users/foo/inbox' + end + end + + context 'with a misconfigured redirection' do + before do + webfinger = { subject: 'acct:Foo@redirected.example.com', links: [{ rel: 'self', href: 'https://ap.example.com/users/foo', type: 'application/activity+json' }] } + stub_request(:get, 'https://redirected.example.com/.well-known/webfinger?resource=acct:Foo@redirected.example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' }) + end + + it 'returns new remote account' do + account = subject.call('Foo@redirected.example.com') + + expect(account.activitypub?).to eq true + expect(account.acct).to eq 'foo@ap.example.com' + expect(account.inbox_url).to eq 'https://ap.example.com/users/foo/inbox' + end + end + + context 'with too many webfinger redirections' do + before do + webfinger = { subject: 'acct:foo@evil.example.com', links: [{ rel: 'self', href: 'https://ap.example.com/users/foo', type: 'application/activity+json' }] } + stub_request(:get, 'https://redirected.example.com/.well-known/webfinger?resource=acct:Foo@redirected.example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' }) + webfinger2 = { subject: 'acct:foo@ap.example.com', links: [{ rel: 'self', href: 'https://ap.example.com/users/foo', type: 'application/activity+json' }] } + stub_request(:get, 'https://evil.example.com/.well-known/webfinger?resource=acct:foo@evil.example.com').to_return(body: Oj.dump(webfinger2), headers: { 'Content-Type': 'application/jrd+json' }) + end + + it 'returns nil' do + expect(subject.call('Foo@redirected.example.com')).to be_nil + end end context 'with an ActivityPub account' do