vbatts/mastodon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add deduplication for JSON payloads in job queue

Browse source
This commit is contained in:
Eugen Rochko 2022-10-25 01:07:00 +02:00
parent 30453fab80
commit 1bfbfb0317
8 changed files with 163 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
app/workers/activitypub/delivery_worker.rb
View file

@ -8,7 +8,7 @@ class ActivityPub::DeliveryWorker
STOPLIGHT_FAILURE_THRESHOLD = 10
STOPLIGHT_COOLDOWN = 60
sidekiq_options queue: 'push', retry: 16, dead: false
sidekiq_options queue: 'push', retry: 16, dead: false, deduplicate_arguments: 0
HEADERS = { 'Content-Type' => 'application/activity+json' }.freeze

2
app/workers/activitypub/processing_worker.rb
View file

@ -3,7 +3,7 @@
class ActivityPub::ProcessingWorker
include Sidekiq::Worker
sidekiq_options backtrace: true, retry: 8
sidekiq_options backtrace: true, retry: 8, deduplicate_arguments: 1
def perform(actor_id, body, delivered_to_account_id = nil, actor_type = 'Account')
case actor_type

18
app/workers/concerns/exponential_backoff.rb
View file

@ -4,6 +4,24 @@ module ExponentialBackoff
extend ActiveSupport::Concern
included do
# # | Next retry backoff | Total waiting time
# ---|--------------------|--------------------
# 1 | 32 | 32
# 2 | 320 | 352
# 3 | 1022 | 1374
# 4 | 3060 | 4434
# 5 | 6778 | 11212
# 6 | 16088 | 27300
# 7 | 37742 | 65042
# 8 | 53799 | 118841
# 9 | 105677 | 224518
# 10 | 129972 | 354490
# 11 | 270226 | 624716
# 12 | 301127 | 925843
# 13 | 287711 | 1213554
# 14 | 616223 | 1829777
# 15 | 607261 | 2437038
# 16 | 1161984 | 3599022
sidekiq_retry_in do |count|
15 + 10 * (count**4) + rand(10 * (count**4))
end