Merge branch 'master' into feature-limited-visibility-bearcaps

app/workers/account_deletion_worker.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class AccountDeletionWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: 'pull', lock: :until_executed
+
+  def perform(account_id, options = {})
+    reserve_username = options.with_indifferent_access.fetch(:reserve_username, true)
+    skip_activitypub = options.with_indifferent_access.fetch(:skip_activitypub, false)
+    DeleteAccountService.new.call(Account.find(account_id), reserve_username: reserve_username, skip_activitypub: skip_activitypub, reserve_email: false)
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+end

app/workers/account_merging_worker.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class AccountMergingWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: 'pull'
+
+  def perform(account_id)
+    account = Account.find(account_id)
+
+    return true if account.nil? || account.local?
+
+    Account.where(uri: account.uri).where.not(id: account.id).find_each do |duplicate|
+      account.merge_with!(duplicate)
+      duplicate.destroy
+    end
+  end
+end

app/workers/activitypub/delivery_worker.rb

@@ -2,6 +2,7 @@
 
 class ActivityPub::DeliveryWorker
   include Sidekiq::Worker
+  include RoutingHelper
   include JsonLdHelper
 
   STOPLIGHT_FAILURE_THRESHOLD = 10
@@ -38,9 +39,18 @@ class ActivityPub::DeliveryWorker
     Request.new(:post, @inbox_url, body: @json, http_client: http_client).tap do |request|
       request.on_behalf_of(@source_account, :uri, sign_with: @options[:sign_with])
       request.add_headers(HEADERS)
+      request.add_headers({ 'Collection-Synchronization' => synchronization_header }) if ENV['DISABLE_FOLLOWERS_SYNCHRONIZATION'] != 'true' && @options[:synchronize_followers]
     end
   end
 
+  def synchronization_header
+    "collectionId=\"#{account_followers_url(@source_account)}\", digest=\"#{@source_account.remote_followers_hash(inbox_url_prefix)}\", url=\"#{account_followers_synchronization_url(@source_account)}\""
+  end
+
+  def inbox_url_prefix
+    @inbox_url[/http(s?):\/\/[^\/]+\//]
+  end
+
   def perform_request
     light = Stoplight(@inbox_url) do
       request_pool.with(@host) do |http_client|

app/workers/activitypub/distribution_worker.rb

@@ -45,7 +45,7 @@ class ActivityPub::DistributionWorker
 
   def deliver_to_inboxes!
     ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url|
-      [payload, @account.id, inbox_url]
+      [payload, @account.id, inbox_url, { synchronize_followers: !@status.distributable? }]
     end
   end
 

app/workers/activitypub/followers_synchronization_worker.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class ActivityPub::FollowersSynchronizationWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: 'push', lock: :until_executed
+
+  def perform(account_id, url)
+    @account = Account.find_by(id: account_id)
+    return true if @account.nil?
+
+    ActivityPub::SynchronizeFollowersService.new.call(@account, url)
+  end
+end

app/workers/admin/account_deletion_worker.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class Admin::AccountDeletionWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: 'pull'
+
+  def perform(account_id)
+    DeleteAccountService.new.call(Account.find(account_id), reserve_username: true, reserve_email: true)
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+end

app/workers/admin/suspension_worker.rb

@@ -5,7 +5,9 @@ class Admin::SuspensionWorker
 
   sidekiq_options queue: 'pull'
 
-  def perform(account_id, remove_user = false)
-    SuspendAccountService.new.call(Account.find(account_id), reserve_username: true, reserve_email: !remove_user)
+  def perform(account_id)
+    SuspendAccountService.new.call(Account.find(account_id))
+  rescue ActiveRecord::RecordNotFound
+    true
   end
 end

app/workers/admin/unsuspension_worker.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class Admin::UnsuspensionWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: 'pull'
+
+  def perform(account_id)
+    UnsuspendAccountService.new.call(Account.find(account_id))
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+end

app/workers/authorize_follow_worker.rb

@@ -7,7 +7,7 @@ class AuthorizeFollowWorker
     source_account = Account.find(source_account_id)
     target_account = Account.find(target_account_id)
 
-    AuthorizeFollowService.new.call(source_account, target_account)
+    AuthorizeFollowService.new.call(source_account, target_account, bypass_limit: true)
   rescue ActiveRecord::RecordNotFound
     true
   end

app/workers/cache_buster_worker.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class CacheBusterWorker
+  include Sidekiq::Worker
+  include RoutingHelper
+
+  sidekiq_options queue: 'pull'
+
+  def perform(path)
+    cache_buster.bust(full_asset_url(path))
+  end
+
+  private
+
+  def cache_buster
+    CacheBuster.new(Rails.configuration.x.cache_buster)
+  end
+end

app/workers/delete_mute_worker.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class DeleteMuteWorker
+  include Sidekiq::Worker
+
+  def perform(mute_id)
+    mute = Mute.find_by(id: mute_id)
+    UnmuteService.new.call(mute.account, mute.target_account) if mute&.expired?
+  end
+end

app/workers/feed_insert_worker.rb

@@ -23,13 +23,25 @@ class FeedInsertWorker
   private
 
   def check_and_insert
-    perform_push unless feed_filtered?
+    return if feed_filtered?
+
+    perform_push
+    perform_notify if notify?
   end
 
   def feed_filtered?
-    # Note: Lists are a variation of home, so the filtering rules
-    # of home apply to both
-    FeedManager.instance.filter?(:home, @status, @follower.id)
+    case @type
+    when :home
+      FeedManager.instance.filter?(:home, @status, @follower)
+    when :list
+      FeedManager.instance.filter?(:list, @status, @list)
+    end
+  end
+
+  def notify?
+    return false if @type != :home || @status.reblog? || (@status.reply? && @status.in_reply_to_account_id != @status.account_id)
+
+    Follow.find_by(account: @follower, target_account: @status.account)&.notify?
   end
 
   def perform_push
@@ -40,4 +52,8 @@ class FeedInsertWorker
       FeedManager.instance.push_to_list(@list, @status)
     end
   end
+
+  def perform_notify
+    NotifyService.new.call(@follower, :status, @status)
+  end
 end

app/workers/import/relationship_worker.rb

@@ -15,7 +15,11 @@ class Import::RelationshipWorker
 
     case relationship
     when 'follow'
-      FollowService.new.call(from_account, target_account, options)
+      begin
+        FollowService.new.call(from_account, target_account, options)
+      rescue ActiveRecord::RecordInvalid
+        raise if FollowLimitValidator.limit_for_account(from_account) < from_account.following_count
+      end
     when 'unfollow'
       UnfollowService.new.call(from_account, target_account)
     when 'block'

app/workers/local_notification_worker.rb

@@ -3,7 +3,7 @@
 class LocalNotificationWorker
   include Sidekiq::Worker
 
-  def perform(receiver_account_id, activity_id = nil, activity_class_name = nil)
+  def perform(receiver_account_id, activity_id = nil, activity_class_name = nil, type = nil)
     if activity_id.nil? && activity_class_name.nil?
       activity = Mention.find(receiver_account_id)
       receiver = activity.account
@@ -12,7 +12,7 @@ class LocalNotificationWorker
       activity = activity_class_name.constantize.find(activity_id)
     end
 
-    NotifyService.new.call(receiver, activity)
+    NotifyService.new.call(receiver, type || activity_class_name.underscore, activity)
   rescue ActiveRecord::RecordNotFound
     true
   end

app/workers/merge_worker.rb

@@ -6,6 +6,8 @@ class MergeWorker
   sidekiq_options queue: 'pull'
 
   def perform(from_account_id, into_account_id)
-    FeedManager.instance.merge_into_timeline(Account.find(from_account_id), Account.find(into_account_id))
+    FeedManager.instance.merge_into_home(Account.find(from_account_id), Account.find(into_account_id))
+  rescue ActiveRecord::RecordNotFound
+    true
   end
 end

app/workers/mute_worker.rb

@@ -4,9 +4,8 @@ class MuteWorker
   include Sidekiq::Worker
 
   def perform(account_id, target_account_id)
-    FeedManager.instance.clear_from_timeline(
-      Account.find(account_id),
-      Account.find(target_account_id)
-    )
+    FeedManager.instance.clear_from_home(Account.find(account_id), Account.find(target_account_id))
+  rescue ActiveRecord::RecordNotFound
+    true
   end
 end

app/workers/poll_expiration_notify_worker.rb

@@ -11,12 +11,12 @@ class PollExpirationNotifyWorker
     # Notify poll owner and remote voters
     if poll.local?
       ActivityPub::DistributePollUpdateWorker.perform_async(poll.status.id)
-      NotifyService.new.call(poll.account, poll)
+      NotifyService.new.call(poll.account, :poll, poll)
     end
 
     # Notify local voters
-    poll.votes.includes(:account).map(&:account).select(&:local?).each do |account|
-      NotifyService.new.call(account, poll)
+    poll.votes.includes(:account).group(:account_id).select(:account_id).map(&:account).select(&:local?).each do |account|
+      NotifyService.new.call(account, :poll, poll)
     end
   rescue ActiveRecord::RecordNotFound
     true

app/workers/refollow_worker.rb

@@ -11,6 +11,7 @@ class RefollowWorker
 
     target_account.passive_relationships.where(account: Account.where(domain: nil)).includes(:account).reorder(nil).find_each do |follow|
       reblogs = follow.show_reblogs?
+      notify  = follow.notify?
 
       # Locally unfollow remote account
       follower = follow.account
@@ -18,7 +19,7 @@ class RefollowWorker
 
       # Schedule re-follow
       begin
-        FollowService.new.call(follower, target_account, reblogs: reblogs)
+        FollowService.new.call(follower, target_account, reblogs: reblogs, notify: notify, bypass_limit: true)
       rescue Mastodon::NotPermittedError, ActiveRecord::RecordNotFound, Mastodon::UnexpectedResponseError, HTTP::Error, OpenSSL::SSL::SSLError
         next
       end

app/workers/scheduler/feed_cleanup_scheduler.rb

@@ -14,37 +14,11 @@ class Scheduler::FeedCleanupScheduler
   private
 
   def clean_home_feeds!
-    clean_feeds!(inactive_account_ids, :home)
+    feed_manager.clean_feeds!(:home, inactive_account_ids)
   end
 
   def clean_list_feeds!
-    clean_feeds!(inactive_list_ids, :list)
-  end
-
-  def clean_feeds!(ids, type)
-    reblogged_id_sets = {}
-
-    redis.pipelined do
-      ids.each do |feed_id|
-        redis.del(feed_manager.key(type, feed_id))
-        reblog_key = feed_manager.key(type, feed_id, 'reblogs')
-        # We collect a future for this: we don't block while getting
-        # it, but we can iterate over it later.
-        reblogged_id_sets[feed_id] = redis.zrange(reblog_key, 0, -1)
-        redis.del(reblog_key)
-      end
-    end
-
-    # Remove all of the reblog tracking keys we just removed the
-    # references to.
-    redis.pipelined do
-      reblogged_id_sets.each do |feed_id, future|
-        future.value.each do |reblogged_id|
-          reblog_set_key = feed_manager.key(type, feed_id, "reblogs:#{reblogged_id}")
-          redis.del(reblog_set_key)
-        end
-      end
-    end
+    feed_manager.clean_feeds!(:list, inactive_list_ids)
   end
 
   def inactive_account_ids

app/workers/scheduler/instance_refresh_scheduler.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class Scheduler::InstanceRefreshScheduler
+  include Sidekiq::Worker
+
+  sidekiq_options lock: :until_executed, retry: 0
+
+  def perform
+    Instance.refresh
+  end
+end

app/workers/scheduler/ip_cleanup_scheduler.rb

@@ -3,13 +3,23 @@
 class Scheduler::IpCleanupScheduler
   include Sidekiq::Worker
 
-  RETENTION_PERIOD = 1.year
+  IP_RETENTION_PERIOD = 1.year.freeze
 
   sidekiq_options lock: :until_executed, retry: 0
 
   def perform
-    time_ago = RETENTION_PERIOD.ago
-    SessionActivation.where('updated_at < ?', time_ago).in_batches.destroy_all
-    User.where('last_sign_in_at < ?', time_ago).where.not(last_sign_in_ip: nil).in_batches.update_all(last_sign_in_ip: nil)
+    clean_ip_columns!
+    clean_expired_ip_blocks!
+  end
+
+  private
+
+  def clean_ip_columns!
+    SessionActivation.where('updated_at < ?', IP_RETENTION_PERIOD.ago).in_batches.destroy_all
+    User.where('current_sign_in_at < ?', IP_RETENTION_PERIOD.ago).in_batches.update_all(last_sign_in_ip: nil, current_sign_in_ip: nil, sign_up_ip: nil)
+  end
+
+  def clean_expired_ip_blocks!
+    IpBlock.expired.in_batches.destroy_all
   end
 end

app/workers/scheduler/user_cleanup_scheduler.rb

@@ -6,9 +6,22 @@ class Scheduler::UserCleanupScheduler
   sidekiq_options lock: :until_executed, retry: 0
 
   def perform
+    clean_unconfirmed_accounts!
+    clean_suspended_accounts!
+  end
+
+  private
+
+  def clean_unconfirmed_accounts!
     User.where('confirmed_at is NULL AND confirmation_sent_at <= ?', 2.days.ago).reorder(nil).find_in_batches do |batch|
       Account.where(id: batch.map(&:account_id)).delete_all
       User.where(id: batch.map(&:id)).delete_all
     end
   end
+
+  def clean_suspended_accounts!
+    AccountDeletionRequest.where('created_at <= ?', AccountDeletionRequest::DELAY_TO_DELETION.ago).reorder(nil).find_each do |deletion_request|
+      Admin::AccountDeletionWorker.perform_async(deletion_request.account_id)
+    end
+  end
 end

app/workers/unfollow_follow_worker.rb

@@ -10,10 +10,11 @@ class UnfollowFollowWorker
     old_target_account = Account.find(old_target_account_id)
     new_target_account = Account.find(new_target_account_id)
 
-    follow = follower_account.active_relationships.find_by(target_account: old_target_account)
+    follow  = follower_account.active_relationships.find_by(target_account: old_target_account)
     reblogs = follow&.show_reblogs?
+    notify  = follow&.notify?
 
-    FollowService.new.call(follower_account, new_target_account, reblogs: reblogs, bypass_locked: bypass_locked)
+    FollowService.new.call(follower_account, new_target_account, reblogs: reblogs, notify: notify, bypass_locked: bypass_locked, bypass_limit: true)
     UnfollowService.new.call(follower_account, old_target_account, skip_unmerge: true)
   rescue ActiveRecord::RecordNotFound, Mastodon::NotPermittedError
     true

app/workers/unmerge_worker.rb

@@ -6,6 +6,8 @@ class UnmergeWorker
   sidekiq_options queue: 'pull'
 
   def perform(from_account_id, into_account_id)
-    FeedManager.instance.unmerge_from_timeline(Account.find(from_account_id), Account.find(into_account_id))
+    FeedManager.instance.unmerge_from_home(Account.find(from_account_id), Account.find(into_account_id))
+  rescue ActiveRecord::RecordNotFound
+    true
   end
 end