diff --git a/Changelog.md b/Changelog.md index e6779a10b..c7a44d674 100644 --- a/Changelog.md +++ b/Changelog.md @@ -59,6 +59,24 @@ wiki](https://wiki.diasporafoundation.org/Integration/Chat#Vines_to_Prosody) for more information on how to migrate to Prosody if you've been using Vines before. +## Sidekiq queue changes + +We've decreased the amount of sidekiq queues from 13 to 5 in PR [#6950](https://github.com/diaspora/diaspora/pull/6950). +The new queues are organized according to priority for the jobs they will process. When upgrading please make sure to +empty the sidekiq queues before shutting down the server for an update. + +If you run your sidekiq with a custom queue configuration, please make sure to update that for the new queues. + +The new queues are: `urgent, high, medium, low, default`. + +When you upgrade to the new version, some jobs may persist in the old queues. To ensure that jobs to be processed, launch +job processing for old queues using command: +``` +rake migrations:run_legacy_queues +``` + +The command will report queues that still have jobs and launch sidekiq process for that queues. + ## Refactor * Improve bookmarklet [#5904](https://github.com/diaspora/diaspora/pull/5904) * Update listen configuration to listen on unix sockets by default [#5974](https://github.com/diaspora/diaspora/pull/5974) @@ -118,6 +136,7 @@ before. * Extract relayable signatures into their own tables [#6932](https://github.com/diaspora/diaspora/pull/6932) * Remove outdated columns from posts table [#6940](https://github.com/diaspora/diaspora/pull/6940) * Remove some unused routes [#6781](https://github.com/diaspora/diaspora/pull/6781) +* Consolidate sidekiq queues [#6950](https://github.com/diaspora/diaspora/pull/6950) ## Bug fixes * Destroy Participation when removing interactions with a post [#5852](https://github.com/diaspora/diaspora/pull/5852) diff --git a/app/workers/clean_cached_files.rb b/app/workers/clean_cached_files.rb index 3e20e4a2c..ddf2e93d3 100644 --- a/app/workers/clean_cached_files.rb +++ b/app/workers/clean_cached_files.rb @@ -1,6 +1,6 @@ module Workers class CleanCachedFiles < Base - sidekiq_options queue: :maintenance + sidekiq_options queue: :low def perform CarrierWave.clean_cached_files! diff --git a/app/workers/deferred_dispatch.rb b/app/workers/deferred_dispatch.rb index 47e2ce7db..49d0aa365 100644 --- a/app/workers/deferred_dispatch.rb +++ b/app/workers/deferred_dispatch.rb @@ -4,7 +4,7 @@ module Workers class DeferredDispatch < Base - sidekiq_options queue: :dispatch + sidekiq_options queue: :high def perform(user_id, object_class_name, object_id, opts) user = User.find(user_id) diff --git a/app/workers/deferred_retraction.rb b/app/workers/deferred_retraction.rb index 43675f5e8..5f3b8834a 100644 --- a/app/workers/deferred_retraction.rb +++ b/app/workers/deferred_retraction.rb @@ -4,7 +4,7 @@ module Workers class DeferredRetraction < Base - sidekiq_options queue: :dispatch + sidekiq_options queue: :high def perform(user_id, retraction_data, recipient_ids, opts) user = User.find(user_id) diff --git a/app/workers/delete_account.rb b/app/workers/delete_account.rb index ef3991f29..e3aa95eff 100644 --- a/app/workers/delete_account.rb +++ b/app/workers/delete_account.rb @@ -5,7 +5,7 @@ module Workers class DeleteAccount < Base - sidekiq_options queue: :delete_account + sidekiq_options queue: :low def perform(account_deletion_id) account_deletion = AccountDeletion.find(account_deletion_id) diff --git a/app/workers/delete_post_from_service.rb b/app/workers/delete_post_from_service.rb index 6ea36c4fc..7d0bfe1fd 100644 --- a/app/workers/delete_post_from_service.rb +++ b/app/workers/delete_post_from_service.rb @@ -4,7 +4,7 @@ # module Workers class DeletePostFromService < Base - sidekiq_options queue: :http_service + sidekiq_options queue: :high def perform(service_id, opts) service = Service.find_by_id(service_id) diff --git a/app/workers/export_photos.rb b/app/workers/export_photos.rb index a0f2471e9..3d036462a 100644 --- a/app/workers/export_photos.rb +++ b/app/workers/export_photos.rb @@ -5,7 +5,7 @@ module Workers class ExportPhotos < Base - sidekiq_options queue: :export + sidekiq_options queue: :low def perform(user_id) @user = User.find(user_id) diff --git a/app/workers/export_user.rb b/app/workers/export_user.rb index 2e1e55aa8..26fadc1fb 100644 --- a/app/workers/export_user.rb +++ b/app/workers/export_user.rb @@ -5,7 +5,7 @@ module Workers class ExportUser < Base - sidekiq_options queue: :export + sidekiq_options queue: :low def perform(user_id) @user = User.find(user_id) diff --git a/app/workers/fetch_profile_photo.rb b/app/workers/fetch_profile_photo.rb index bea7e104c..8eb370ecb 100644 --- a/app/workers/fetch_profile_photo.rb +++ b/app/workers/fetch_profile_photo.rb @@ -5,7 +5,7 @@ module Workers class FetchProfilePhoto < Base - sidekiq_options queue: :photos + sidekiq_options queue: :medium def perform(user_id, service_id, fallback_image_url = nil) service = Service.find(service_id) diff --git a/app/workers/fetch_public_posts.rb b/app/workers/fetch_public_posts.rb index 8b5ca00c8..228efe81e 100644 --- a/app/workers/fetch_public_posts.rb +++ b/app/workers/fetch_public_posts.rb @@ -4,7 +4,7 @@ module Workers class FetchPublicPosts < Base - sidekiq_options queue: :http_service + sidekiq_options queue: :medium def perform(diaspora_id) Diaspora::Fetcher::Public.new.fetch!(diaspora_id) diff --git a/app/workers/fetch_webfinger.rb b/app/workers/fetch_webfinger.rb index 3b277824d..182304ef8 100644 --- a/app/workers/fetch_webfinger.rb +++ b/app/workers/fetch_webfinger.rb @@ -4,7 +4,7 @@ module Workers class FetchWebfinger < Base - sidekiq_options queue: :socket_webfinger + sidekiq_options queue: :urgent def perform(account) person = Person.find_or_fetch_by_identifier(account) diff --git a/app/workers/gather_o_embed_data.rb b/app/workers/gather_o_embed_data.rb index cb0ad0819..e6eb695b2 100644 --- a/app/workers/gather_o_embed_data.rb +++ b/app/workers/gather_o_embed_data.rb @@ -5,7 +5,7 @@ module Workers class GatherOEmbedData < Base - sidekiq_options queue: :http_service + sidekiq_options queue: :medium def perform(post_id, url, retry_count=1) post = Post.find(post_id) diff --git a/app/workers/gather_open_graph_data.rb b/app/workers/gather_open_graph_data.rb index 18c97ab8b..f962e0411 100644 --- a/app/workers/gather_open_graph_data.rb +++ b/app/workers/gather_open_graph_data.rb @@ -5,7 +5,7 @@ module Workers class GatherOpenGraphData < Base - sidekiq_options queue: :http_service + sidekiq_options queue: :medium def perform(post_id, url, retry_count=1) post = Post.find(post_id) diff --git a/app/workers/mail/also_commented.rb b/app/workers/mail/also_commented.rb index 2197c8e25..52026e737 100644 --- a/app/workers/mail/also_commented.rb +++ b/app/workers/mail/also_commented.rb @@ -1,7 +1,7 @@ module Workers module Mail class AlsoCommented < Base - sidekiq_options queue: :mail + sidekiq_options queue: :low def perform(recipient_id, sender_id, comment_id) if email = Notifier.also_commented(recipient_id, sender_id, comment_id) diff --git a/app/workers/mail/comment_on_post.rb b/app/workers/mail/comment_on_post.rb index fc397518d..070eb8e33 100644 --- a/app/workers/mail/comment_on_post.rb +++ b/app/workers/mail/comment_on_post.rb @@ -1,7 +1,7 @@ module Workers module Mail class CommentOnPost < Base - sidekiq_options queue: :mail + sidekiq_options queue: :low def perform(recipient_id, sender_id, comment_id) Notifier.comment_on_post(recipient_id, sender_id, comment_id).deliver_now diff --git a/app/workers/mail/confirm_email.rb b/app/workers/mail/confirm_email.rb index 05a9cef5e..252f030c6 100644 --- a/app/workers/mail/confirm_email.rb +++ b/app/workers/mail/confirm_email.rb @@ -1,7 +1,7 @@ module Workers module Mail class ConfirmEmail < Base - sidekiq_options queue: :mail + sidekiq_options queue: :low def perform(user_id) Notifier.confirm_email(user_id).deliver_now diff --git a/app/workers/mail/invite_email.rb b/app/workers/mail/invite_email.rb index 685d628e5..0b18841d9 100644 --- a/app/workers/mail/invite_email.rb +++ b/app/workers/mail/invite_email.rb @@ -5,7 +5,7 @@ module Workers module Mail class InviteEmail < Base - sidekiq_options queue: :mail + sidekiq_options queue: :low def perform(emails, inviter_id, options={}) EmailInviter.new(emails, User.find(inviter_id), options).send! diff --git a/app/workers/mail/liked.rb b/app/workers/mail/liked.rb index d2b2748c9..595af3e39 100644 --- a/app/workers/mail/liked.rb +++ b/app/workers/mail/liked.rb @@ -1,7 +1,7 @@ module Workers module Mail class Liked < Base - sidekiq_options queue: :mail + sidekiq_options queue: :low def perform(recipient_id, sender_id, like_id) Notifier.liked(recipient_id, sender_id, like_id).deliver_now diff --git a/app/workers/mail/mentioned.rb b/app/workers/mail/mentioned.rb index 06dac676b..a32f30f11 100644 --- a/app/workers/mail/mentioned.rb +++ b/app/workers/mail/mentioned.rb @@ -6,7 +6,7 @@ module Workers module Mail class Mentioned < Base - sidekiq_options queue: :mail + sidekiq_options queue: :low def perform(recipient_id, actor_id, target_id) Notifier.mentioned( recipient_id, actor_id, target_id).deliver_now diff --git a/app/workers/mail/private_message.rb b/app/workers/mail/private_message.rb index 6d48d5dbf..d24ce0296 100644 --- a/app/workers/mail/private_message.rb +++ b/app/workers/mail/private_message.rb @@ -6,7 +6,7 @@ module Workers module Mail class PrivateMessage < Base - sidekiq_options queue: :mail + sidekiq_options queue: :low def perform(recipient_id, actor_id, target_id) Notifier.private_message( recipient_id, actor_id, target_id).deliver_now diff --git a/app/workers/mail/report_worker.rb b/app/workers/mail/report_worker.rb index 2d34787ae..8139266e6 100644 --- a/app/workers/mail/report_worker.rb +++ b/app/workers/mail/report_worker.rb @@ -1,7 +1,7 @@ module Workers module Mail class ReportWorker < Base - sidekiq_options queue: :mail + sidekiq_options queue: :low def perform(report_id) ReportMailer.new_report(report_id).each(&:deliver_now) diff --git a/app/workers/mail/reshared.rb b/app/workers/mail/reshared.rb index 20df36ac0..1144147ad 100644 --- a/app/workers/mail/reshared.rb +++ b/app/workers/mail/reshared.rb @@ -1,7 +1,7 @@ module Workers module Mail class Reshared < Base - sidekiq_options queue: :mail + sidekiq_options queue: :low def perform(recipient_id, sender_id, reshare_id) Notifier.reshared(recipient_id, sender_id, reshare_id).deliver_now diff --git a/app/workers/mail/started_sharing.rb b/app/workers/mail/started_sharing.rb index 3bc4938c8..3618d1527 100644 --- a/app/workers/mail/started_sharing.rb +++ b/app/workers/mail/started_sharing.rb @@ -6,7 +6,7 @@ module Workers module Mail class StartedSharing < Base - sidekiq_options queue: :mail + sidekiq_options queue: :low def perform(recipient_id, sender_id, target_id) Notifier.started_sharing(recipient_id, sender_id).deliver_now diff --git a/app/workers/post_to_service.rb b/app/workers/post_to_service.rb index 24cf053db..933913663 100644 --- a/app/workers/post_to_service.rb +++ b/app/workers/post_to_service.rb @@ -4,7 +4,7 @@ # module Workers class PostToService < Base - sidekiq_options queue: :http_service + sidekiq_options queue: :medium def perform(service_id, post_id, url) service = Service.find_by_id(service_id) diff --git a/app/workers/process_photo.rb b/app/workers/process_photo.rb index 2fcdd3943..1686c5202 100644 --- a/app/workers/process_photo.rb +++ b/app/workers/process_photo.rb @@ -5,7 +5,7 @@ module Workers class ProcessPhoto < Base - sidekiq_options queue: :photos + sidekiq_options queue: :low def perform(id) photo = Photo.find(id) diff --git a/app/workers/publish_to_hub.rb b/app/workers/publish_to_hub.rb index 0e87c7936..7d33daeba 100644 --- a/app/workers/publish_to_hub.rb +++ b/app/workers/publish_to_hub.rb @@ -4,7 +4,7 @@ module Workers class PublishToHub < Base - sidekiq_options queue: :http_service + sidekiq_options queue: :medium def perform(sender_atom_url) Pubsubhubbub.new(AppConfig.environment.pubsub_server.get).publish(sender_atom_url) diff --git a/app/workers/queue_users_for_removal.rb b/app/workers/queue_users_for_removal.rb index bfe3a4e39..87c720b30 100644 --- a/app/workers/queue_users_for_removal.rb +++ b/app/workers/queue_users_for_removal.rb @@ -4,7 +4,7 @@ module Workers class QueueUsersForRemoval < Base - sidekiq_options queue: :maintenance + sidekiq_options queue: :low def perform # Queue users for removal due to inactivity diff --git a/app/workers/receive_base.rb b/app/workers/receive_base.rb index f204fd3b6..15c691d69 100644 --- a/app/workers/receive_base.rb +++ b/app/workers/receive_base.rb @@ -1,6 +1,6 @@ module Workers class ReceiveBase < Base - sidekiq_options queue: :receive + sidekiq_options queue: :urgent include Diaspora::Logging diff --git a/app/workers/receive_local.rb b/app/workers/receive_local.rb index f02cb504e..a93332979 100644 --- a/app/workers/receive_local.rb +++ b/app/workers/receive_local.rb @@ -1,6 +1,6 @@ module Workers class ReceiveLocal < Base - sidekiq_options queue: :receive_local + sidekiq_options queue: :high def perform(object_class_string, object_id, recipient_user_ids) object = object_class_string.constantize.find(object_id) diff --git a/app/workers/recurring_pod_check.rb b/app/workers/recurring_pod_check.rb index 6a6940da4..f6ed65700 100644 --- a/app/workers/recurring_pod_check.rb +++ b/app/workers/recurring_pod_check.rb @@ -1,6 +1,6 @@ module Workers class RecurringPodCheck < Base - sidekiq_options queue: :maintenance + sidekiq_options queue: :low def perform Pod.check_all! diff --git a/app/workers/remove_old_user.rb b/app/workers/remove_old_user.rb index d8619b4bf..444857ec2 100644 --- a/app/workers/remove_old_user.rb +++ b/app/workers/remove_old_user.rb @@ -4,7 +4,7 @@ module Workers class RemoveOldUser < Base - sidekiq_options queue: :maintenance + sidekiq_options queue: :low def safe_remove_after # extra safety time to compare in addition to remove_after @@ -24,4 +24,4 @@ module Workers end end end -end \ No newline at end of file +end diff --git a/app/workers/resend_invitation.rb b/app/workers/resend_invitation.rb index b2a0cc00e..5b698a28d 100644 --- a/app/workers/resend_invitation.rb +++ b/app/workers/resend_invitation.rb @@ -5,7 +5,7 @@ module Workers class ResendInvitation < Base - sidekiq_options queue: :mail + sidekiq_options queue: :low def perform(invitation_id) inv = Invitation.find(invitation_id) diff --git a/app/workers/reset_password.rb b/app/workers/reset_password.rb index 965876e5e..9fdd06bbb 100644 --- a/app/workers/reset_password.rb +++ b/app/workers/reset_password.rb @@ -1,6 +1,6 @@ module Workers class ResetPassword < Base - sidekiq_options queue: :mail + sidekiq_options queue: :urgent def perform(user_id) User.find(user_id).send_reset_password_instructions! diff --git a/app/workers/send_base.rb b/app/workers/send_base.rb index f7cce44a3..9da921e37 100644 --- a/app/workers/send_base.rb +++ b/app/workers/send_base.rb @@ -1,6 +1,6 @@ module Workers class SendBase < Base - sidekiq_options queue: :http, retry: 0 + sidekiq_options queue: :medium, retry: 0 MAX_RETRIES = AppConfig.environment.sidekiq.retry.get.to_i diff --git a/config/sidekiq.yml b/config/sidekiq.yml index f4c29447e..a99dd69c9 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -8,16 +8,8 @@ :dead_max_jobs: <%= AppConfig.environment.sidekiq.dead_jobs_limit.to_i %> :dead_timeout_in_seconds: <%= AppConfig.environment.sidekiq.dead_jobs_timeout.to_i %> :queues: - - socket_webfinger - - photos - - http_service - - dispatch - - mail - - delete_account - - receive_local - - receive - - receive_salmon - - http - - export - - maintenance + - urgent + - high + - medium + - low - default