From 3fc3b249e71f05a664a81d47d35f9b2815de5e97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonne=20Ha=C3=9F?= Date: Tue, 19 Feb 2013 05:37:58 +0100 Subject: [PATCH 1/2] End the require mess * Rename and reorganize post fetcher to fix autoloading, also let it use Faradays default connection so we get nice redirects * Add initializer to load libs at a central place * added lib dir to autoload_once paths to increase thread safety * Moved lib/exceptions.rb to lib/diaspora/ to conform namespacing --- app/controllers/admins_controller.rb | 2 -- app/controllers/invitations_controller.rb | 2 -- app/controllers/likes_controller.rb | 2 -- app/controllers/people_controller.rb | 2 -- app/controllers/posts_controller.rb | 2 -- app/controllers/publics_controller.rb | 4 --- app/controllers/streams_controller.rb | 9 ----- app/controllers/tags_controller.rb | 2 -- app/controllers/users_controller.rb | 3 -- app/helpers/markdownify_helper.rb | 3 -- app/models/jobs/http_multi.rb | 2 +- app/models/person.rb | 3 -- app/models/photo.rb | 2 -- app/models/service.rb | 5 ++- app/models/services/facebook.rb | 2 -- app/models/services/twitter.rb | 2 -- app/models/status_message.rb | 1 - app/models/user.rb | 3 -- app/models/user/querying.rb | 3 -- app/presenters/o_embed_presenter.rb | 1 - app/presenters/post_presenter.rb | 2 -- config/application.rb | 4 +-- config/environment.rb | 2 -- config/initializers/acts_as_taggable_on.rb | 3 +- config/initializers/load_libraries.rb | 36 +++++++++++++++++++ features/step_definitions/debug_steps.rb | 2 +- features/step_definitions/web_steps.rb | 2 -- lib/diaspora.rb | 6 +++- lib/{ => diaspora}/exceptions.rb | 0 lib/diaspora/federated/base.rb | 2 -- lib/diaspora/fetcher.rb | 5 +++ lib/diaspora/fetcher/public.rb | 36 ++++++++----------- lib/diaspora/markdownify.rb | 2 -- lib/diaspora/markdownify_email.rb | 4 +-- lib/diaspora/taggable.rb | 2 -- lib/{hcard.rb => h_card.rb} | 0 lib/hydra_wrapper.rb | 3 -- lib/postzord.rb | 3 +- lib/postzord/dispatcher.rb | 4 +-- lib/postzord/receiver.rb | 5 +-- lib/postzord/receiver/private.rb | 3 -- lib/{salmon => }/salmon.rb | 6 ++-- lib/stream.rb | 11 +++++- lib/stream/base.rb | 1 - lib/webfinger.rb | 3 -- spec/lib/diaspora/fetcher/public_spec.rb | 36 ++++++++++--------- spec/lib/email_inviter_spec.rb | 4 +-- spec/lib/hcard_spec.rb | 1 - spec/lib/postzord/dispatcher/private_spec.rb | 1 - spec/lib/postzord/dispatcher_spec.rb | 2 -- .../lib/postzord/receiver/local_batch_spec.rb | 1 - spec/lib/postzord/receiver/private_spec.rb | 3 -- spec/lib/postzord/receiver/public_spec.rb | 3 -- spec/lib/postzord/receiver_spec.rb | 2 -- spec/lib/publisher_spec.rb | 3 -- spec/lib/pubsubhubbub_spec.rb | 3 +- spec/lib/rake_helper_spec.rb | 3 +- spec/lib/statistics_spec.rb | 1 - spec/lib/stream/base_spec.rb | 1 + spec/lib/template_picker_spec.rb | 2 +- spec/models/jobs/publish_to_hub_spec.rb | 1 - 61 files changed, 115 insertions(+), 154 deletions(-) create mode 100644 config/initializers/load_libraries.rb rename lib/{ => diaspora}/exceptions.rb (100%) create mode 100644 lib/diaspora/fetcher.rb rename lib/{hcard.rb => h_card.rb} (100%) rename lib/{salmon => }/salmon.rb (76%) diff --git a/app/controllers/admins_controller.rb b/app/controllers/admins_controller.rb index 36a5ef2f8..e8b0a0aa6 100644 --- a/app/controllers/admins_controller.rb +++ b/app/controllers/admins_controller.rb @@ -1,5 +1,3 @@ -require Rails.root.join('lib', 'statistics') - class AdminsController < ApplicationController before_filter :authenticate_user! before_filter :redirect_unless_admin diff --git a/app/controllers/invitations_controller.rb b/app/controllers/invitations_controller.rb index 984c6ff5e..3754d1b22 100644 --- a/app/controllers/invitations_controller.rb +++ b/app/controllers/invitations_controller.rb @@ -2,8 +2,6 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -require Rails.root.join('lib', 'email_inviter') - class InvitationsController < ApplicationController before_filter :authenticate_user!, :only => [:new, :create] diff --git a/app/controllers/likes_controller.rb b/app/controllers/likes_controller.rb index 7be46c166..8ab65b6ca 100644 --- a/app/controllers/likes_controller.rb +++ b/app/controllers/likes_controller.rb @@ -2,8 +2,6 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -require Rails.root.join("app", "presenters", "post_presenter") - class LikesController < ApplicationController include ApplicationHelper before_filter :authenticate_user! diff --git a/app/controllers/people_controller.rb b/app/controllers/people_controller.rb index 3115f1f1b..d11742ec3 100644 --- a/app/controllers/people_controller.rb +++ b/app/controllers/people_controller.rb @@ -2,8 +2,6 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -require Rails.root.join("lib", 'stream', "person") - class PeopleController < ApplicationController before_filter :authenticate_user!, :except => [:show, :last_post] before_filter :redirect_if_tag_search, :only => [:index] diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index e901da938..10c579337 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -2,8 +2,6 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -require Rails.root.join("app", "presenters", "post_presenter") - class PostsController < ApplicationController include PostsHelper diff --git a/app/controllers/publics_controller.rb b/app/controllers/publics_controller.rb index 31942988e..646e68480 100644 --- a/app/controllers/publics_controller.rb +++ b/app/controllers/publics_controller.rb @@ -2,11 +2,7 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -require Rails.root.join('lib', 'stream', 'public') class PublicsController < ApplicationController - require Rails.root.join('lib', 'diaspora', 'parser') - require Rails.root.join('lib', 'postzord', 'receiver', 'public') - require Rails.root.join('lib', 'postzord', 'receiver', 'private') include Diaspora::Parser skip_before_filter :set_header_data diff --git a/app/controllers/streams_controller.rb b/app/controllers/streams_controller.rb index de510d1ed..7c4484f51 100644 --- a/app/controllers/streams_controller.rb +++ b/app/controllers/streams_controller.rb @@ -2,15 +2,6 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -require Rails.root.join("lib", "stream", "aspect") -require Rails.root.join("lib", "stream", "multi") -require Rails.root.join("lib", "stream", "comments") -require Rails.root.join("lib", "stream", "likes") -require Rails.root.join("lib", "stream", "mention") -require Rails.root.join("lib", "stream", "followed_tag") -require Rails.root.join("lib", "stream", "activity") - - class StreamsController < ApplicationController before_filter :authenticate_user! before_filter :save_selected_aspects, :only => :aspects diff --git a/app/controllers/tags_controller.rb b/app/controllers/tags_controller.rb index 13766ac13..9d86638fa 100644 --- a/app/controllers/tags_controller.rb +++ b/app/controllers/tags_controller.rb @@ -1,8 +1,6 @@ # Copyright (c) 2010-2011, Diaspora Inc. This file is # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -require Rails.root.join('app', 'models', 'acts_as_taggable_on', 'tag') -require Rails.root.join('lib', 'stream', 'tag') class TagsController < ApplicationController skip_before_filter :set_grammatical_gender diff --git a/app/controllers/users_controller.rb b/app/controllers/users_controller.rb index f3fb55e4b..528baa33f 100644 --- a/app/controllers/users_controller.rb +++ b/app/controllers/users_controller.rb @@ -3,9 +3,6 @@ # the COPYRIGHT file. class UsersController < ApplicationController - require Rails.root.join('lib', 'diaspora', 'exporter') - require Rails.root.join('lib', 'collect_user_photos') - before_filter :authenticate_user!, :except => [:new, :create, :public, :user_photo] respond_to :html diff --git a/app/helpers/markdownify_helper.rb b/app/helpers/markdownify_helper.rb index 926029077..f9ff0f73c 100644 --- a/app/helpers/markdownify_helper.rb +++ b/app/helpers/markdownify_helper.rb @@ -2,9 +2,6 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -require Rails.root.join('lib', 'diaspora', 'markdownify') -require 'redcarpet/render_strip' - module MarkdownifyHelper def markdownify(target, render_options={}) diff --git a/app/models/jobs/http_multi.rb b/app/models/jobs/http_multi.rb index 8d686c233..1132fcbe4 100644 --- a/app/models/jobs/http_multi.rb +++ b/app/models/jobs/http_multi.rb @@ -17,7 +17,7 @@ module Jobs people = Person.where(:id => person_ids) dispatcher = dispatcher_class_as_string.constantize - hydra = HydraWrapper.new(user, people, encoded_object_xml, dispatcher) + hydra = ::HydraWrapper.new(user, people, encoded_object_xml, dispatcher) hydra.enqueue_batch hydra.run diff --git a/app/models/person.rb b/app/models/person.rb index 4fe1fd9fb..0e1dbb40c 100644 --- a/app/models/person.rb +++ b/app/models/person.rb @@ -2,9 +2,6 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -require 'uri' -require Rails.root.join('lib', 'hcard') - class Person < ActiveRecord::Base include ROXML include Encryptor::Public diff --git a/app/models/photo.rb b/app/models/photo.rb index 9741985a7..fdcabedcc 100644 --- a/app/models/photo.rb +++ b/app/models/photo.rb @@ -3,8 +3,6 @@ # the COPYRIGHT file. class Photo < ActiveRecord::Base - require 'carrierwave/orm/activerecord' - include Diaspora::Federated::Shareable include Diaspora::Commentable include Diaspora::Shareable diff --git a/app/models/service.rb b/app/models/service.rb index a4029a370..beb41fa96 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -4,7 +4,6 @@ class Service < ActiveRecord::Base include ActionView::Helpers::TextHelper - require Rails.root.join('app', 'helpers', 'markdownify_helper') include MarkdownifyHelper belongs_to :user @@ -39,5 +38,5 @@ class Service < ActiveRecord::Base end end -require Rails.root.join('app', 'models', 'services', 'facebook') -require Rails.root.join('app', 'models', 'services', 'twitter') +require 'services/facebook' +require 'services/twitter' diff --git a/app/models/services/facebook.rb b/app/models/services/facebook.rb index d5a6fb633..93ba07ef5 100644 --- a/app/models/services/facebook.rb +++ b/app/models/services/facebook.rb @@ -1,7 +1,5 @@ -require 'uri' class Services::Facebook < Service include Rails.application.routes.url_helpers - require Rails.root.join('app', 'helpers', 'markdownify_helper') include MarkdownifyHelper OVERRIDE_FIELDS_ON_FB_UPDATE = [:contact_id, :person_id, :request_id, :invitation_id, :photo_url, :name, :username] diff --git a/app/models/services/twitter.rb b/app/models/services/twitter.rb index 0fa79360d..f874ecb48 100644 --- a/app/models/services/twitter.rb +++ b/app/models/services/twitter.rb @@ -1,5 +1,3 @@ -require 'uri' - class Services::Twitter < Service MAX_CHARACTERS = 140 SHORTENED_URL_LENGTH = 21 diff --git a/app/models/status_message.rb b/app/models/status_message.rb index e6c46cd4d..03f001643 100644 --- a/app/models/status_message.rb +++ b/app/models/status_message.rb @@ -159,7 +159,6 @@ class StatusMessage < Post end def contains_oembed_url_in_text? - require 'uri' urls = URI.extract(self.raw_message, ['http', 'https']) self.oembed_url = urls.find{ |url| !TRUSTED_OEMBED_PROVIDERS.find(url).nil? } end diff --git a/app/models/user.rb b/app/models/user.rb index 4f356717b..32656c0ed 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -2,9 +2,6 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -require Rails.root.join('lib', 'salmon', 'salmon') -require Rails.root.join('lib', 'postzord', 'dispatcher') - class User < ActiveRecord::Base include Encryptor::Private include Connecting diff --git a/app/models/user/querying.rb b/app/models/user/querying.rb index 3816b434a..c849b13a7 100644 --- a/app/models/user/querying.rb +++ b/app/models/user/querying.rb @@ -2,9 +2,6 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -require Rails.root.join('lib', 'evil_query') - - #TODO: THIS FILE SHOULD NOT EXIST, EVIL SQL SHOULD BE ENCAPSULATED IN EvilQueries, #throwing all of this stuff in user violates demeter like WHOA module User::Querying diff --git a/app/presenters/o_embed_presenter.rb b/app/presenters/o_embed_presenter.rb index c8baf4752..a3e875d30 100644 --- a/app/presenters/o_embed_presenter.rb +++ b/app/presenters/o_embed_presenter.rb @@ -1,4 +1,3 @@ -require 'uri' class OEmbedPresenter include PostsHelper include ActionView::Helpers::TextHelper diff --git a/app/presenters/post_presenter.rb b/app/presenters/post_presenter.rb index a0558bc4e..895ff9a4f 100644 --- a/app/presenters/post_presenter.rb +++ b/app/presenters/post_presenter.rb @@ -1,5 +1,3 @@ -require Rails.root.join('lib', 'template_picker') - class PostPresenter attr_accessor :post, :current_user diff --git a/config/application.rb b/config/application.rb index f8b35569c..0a5e7c510 100644 --- a/config/application.rb +++ b/config/application.rb @@ -32,8 +32,8 @@ module Diaspora # -- all .rb files in that directory are automatically loaded. # Add additional load paths for your own custom dirs - config.autoload_paths += %W(#{config.root}/app/presenters) - config.autoload_paths += %W(#{config.root}/lib) + config.autoload_paths += %W{#{config.root}/app/presenters} + config.autoload_once_paths += %W{#{config.root}/lib} # Only load the plugins named here, in the order given (default is alphabetical). # :all can be used as a placeholder for all plugins not explicitly named diff --git a/config/environment.rb b/config/environment.rb index 57ad7ff2e..ad18fc67e 100644 --- a/config/environment.rb +++ b/config/environment.rb @@ -13,7 +13,6 @@ end # Load the rails application require Pathname.new(__FILE__).dirname.expand_path.join('application') -require Rails.root.join("lib", "exceptions") # Load configuration system early require Rails.root.join('config', 'load_config') @@ -27,7 +26,6 @@ USERNAME_BLACKLIST = ['admin', 'administrator', 'hostmaster', 'info', 'postmaste # Initialize the rails application Diaspora::Application.initialize! -require Rails.root.join('lib', 'federation_logger') # allow token auth only for posting activitystream photos module Devise diff --git a/config/initializers/acts_as_taggable_on.rb b/config/initializers/acts_as_taggable_on.rb index 1160cfafd..d42a381b1 100644 --- a/config/initializers/acts_as_taggable_on.rb +++ b/config/initializers/acts_as_taggable_on.rb @@ -1,3 +1,2 @@ - ActsAsTaggableOn.force_lowercase = true - +require Rails.root.join("app", "models", "acts_as_taggable_on", "tag") diff --git a/config/initializers/load_libraries.rb b/config/initializers/load_libraries.rb new file mode 100644 index 000000000..4aaa7d6d5 --- /dev/null +++ b/config/initializers/load_libraries.rb @@ -0,0 +1,36 @@ +# Stdlib +require 'cgi' +require 'uri' + +# Not auto required gems +require 'active_support/base64' +require 'builder/xchar' +require 'carrierwave/orm/activerecord' +require 'erb' +require 'redcarpet/render_strip' +require 'typhoeus' + +# Presenters +require 'post_presenter' + +# Helpers +require 'markdownify_helper' + +# Our libs +require 'collect_user_photos' +require 'diaspora' +require 'email_inviter' +require 'evil_query' +require 'federation_logger' +require 'h_card' +require 'hydra_wrapper' +require 'postzord' +require 'publisher' +require 'pubsubhubbub' +require 'salmon' +require 'statistics' +require 'stream' +require 'template_picker' +require 'webfinger' +require 'webfinger_profile' + diff --git a/features/step_definitions/debug_steps.rb b/features/step_definitions/debug_steps.rb index 58a36d533..bf255c633 100644 --- a/features/step_definitions/debug_steps.rb +++ b/features/step_definitions/debug_steps.rb @@ -1,6 +1,6 @@ module DebuggingCukeHelpers def start_debugging - require 'ruby-debug' + require 'debug' debugger true end diff --git a/features/step_definitions/web_steps.rb b/features/step_definitions/web_steps.rb index 62fcebb62..7be28c6cf 100644 --- a/features/step_definitions/web_steps.rb +++ b/features/step_definitions/web_steps.rb @@ -4,8 +4,6 @@ # instead of editing this one. Cucumber will automatically load all features/**/*.rb # files. -require 'uri' -require 'cgi' require File.expand_path(File.join(File.dirname(__FILE__), "..", "support", "paths")) module WithinHelpers diff --git a/lib/diaspora.rb b/lib/diaspora.rb index 18d812247..de8d1d003 100644 --- a/lib/diaspora.rb +++ b/lib/diaspora.rb @@ -3,5 +3,9 @@ # the COPYRIGHT file. module Diaspora - autoload :Parser + require 'diaspora/exceptions' + require 'diaspora/parser' + require 'diaspora/fetcher' + require 'diaspora/markdownify' + require 'diaspora/exporter' end diff --git a/lib/exceptions.rb b/lib/diaspora/exceptions.rb similarity index 100% rename from lib/exceptions.rb rename to lib/diaspora/exceptions.rb diff --git a/lib/diaspora/federated/base.rb b/lib/diaspora/federated/base.rb index 32054cca6..52ebcc858 100644 --- a/lib/diaspora/federated/base.rb +++ b/lib/diaspora/federated/base.rb @@ -9,8 +9,6 @@ #including this module lets you federate an object at the most basic of level -require 'builder/xchar' - module Diaspora module Federated module Base diff --git a/lib/diaspora/fetcher.rb b/lib/diaspora/fetcher.rb new file mode 100644 index 000000000..adb82e552 --- /dev/null +++ b/lib/diaspora/fetcher.rb @@ -0,0 +1,5 @@ +module Diaspora + module Fetcher + require 'diaspora/fetcher/public' + end +end diff --git a/lib/diaspora/fetcher/public.rb b/lib/diaspora/fetcher/public.rb index 501f3e5b7..409d3ffe8 100644 --- a/lib/diaspora/fetcher/public.rb +++ b/lib/diaspora/fetcher/public.rb @@ -1,8 +1,7 @@ # Copyright (c) 2010-2012, Diaspora Inc. This file is # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. - -class PublicFetcher +module Diaspora; module Fetcher; class Public # various states that can be assigned to a person to describe where # in the process of fetching their public posts we're currently at @@ -23,11 +22,11 @@ class PublicFetcher begin retrieve_and_process_posts rescue => e - set_fetch_status PublicFetcher::Status_Failed + set_fetch_status Public::Status_Failed raise e end - set_fetch_status PublicFetcher::Status_Done + set_fetch_status Public::Status_Done end private @@ -35,20 +34,20 @@ class PublicFetcher # if it is reasonable to do so, and that they have not been fetched already def qualifies_for_fetching? raise ActiveRecord::RecordNotFound unless @person.present? - return false if @person.fetch_status == PublicFetcher::Status_Unfetchable + return false if @person.fetch_status == Public::Status_Unfetchable # local users don't need to be fetched if @person.local? - set_fetch_status PublicFetcher::Status_Unfetchable + set_fetch_status Public::Status_Unfetchable return false end # this record is already being worked on - return false if @person.fetch_status > PublicFetcher::Status_Initial + return false if @person.fetch_status > Public::Status_Initial # ok, let's go @person.remote? && - @person.fetch_status == PublicFetcher::Status_Initial + @person.fetch_status == Public::Status_Initial end # call the methods to fetch and process the public posts for the person @@ -72,24 +71,19 @@ class PublicFetcher # fetch the public posts of the person from their server and save the # JSON response to `@data` def retrieve_posts - set_fetch_status PublicFetcher::Status_Running + set_fetch_status Public::Status_Running FEDERATION_LOGGER.info "fetching public posts for #{@person.diaspora_handle}" - conn = Faraday.new(:url => @person.url) do |c| - c.request :json - c.response :json - c.adapter :net_http + resp = Faraday.get("#{@person.url}/people/#{@person.guid}") do |req| + req.headers[:accept] = 'application/json' + req.headers[:user_agent] = 'diaspora-fetcher' end - conn.headers[:user_agent] = 'diaspora-fetcher' - conn.headers[:accept] = 'application/json' - - resp = conn.get "/people/#{@person.guid}" FEDERATION_LOGGER.debug resp.body.to_s[0..250] - @data = resp.body - set_fetch_status PublicFetcher::Status_Fetched + @data = JSON.parse resp.body + set_fetch_status Public::Status_Fetched end # process the public posts that were previously fetched with `retrieve_posts` @@ -124,7 +118,7 @@ class PublicFetcher StatusMessage.set_callback :create, :set_guid end - set_fetch_status PublicFetcher::Status_Processed + set_fetch_status Public::Status_Processed end # set and save the fetch status for the current person @@ -182,4 +176,4 @@ class PublicFetcher type_ok end -end +end; end; end diff --git a/lib/diaspora/markdownify.rb b/lib/diaspora/markdownify.rb index e8001ba8f..c31ddbc0c 100644 --- a/lib/diaspora/markdownify.rb +++ b/lib/diaspora/markdownify.rb @@ -1,5 +1,3 @@ -require 'erb' - module Diaspora module Markdownify class HTML < Redcarpet::Render::HTML diff --git a/lib/diaspora/markdownify_email.rb b/lib/diaspora/markdownify_email.rb index bdc7983ba..1e3263127 100644 --- a/lib/diaspora/markdownify_email.rb +++ b/lib/diaspora/markdownify_email.rb @@ -1,5 +1,3 @@ -require Rails.root.join("app", "models", "acts_as_taggable_on", "tag") - module Diaspora module Markdownify class Email < Redcarpet::Render::HTML @@ -26,4 +24,4 @@ module Diaspora end end end -end \ No newline at end of file +end diff --git a/lib/diaspora/taggable.rb b/lib/diaspora/taggable.rb index b55cb021a..13c9bb811 100644 --- a/lib/diaspora/taggable.rb +++ b/lib/diaspora/taggable.rb @@ -2,8 +2,6 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -require Rails.root.join("app", "models", "acts_as_taggable_on", "tag") - module Diaspora module Taggable def self.included(model) diff --git a/lib/hcard.rb b/lib/h_card.rb similarity index 100% rename from lib/hcard.rb rename to lib/h_card.rb diff --git a/lib/hydra_wrapper.rb b/lib/hydra_wrapper.rb index 627ac3ba7..a2d02307a 100644 --- a/lib/hydra_wrapper.rb +++ b/lib/hydra_wrapper.rb @@ -2,9 +2,6 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -require 'typhoeus' -require 'active_support/base64' - class HydraWrapper OPTS = {:max_redirects => 3, :timeout => 25000, :method => :post, diff --git a/lib/postzord.rb b/lib/postzord.rb index 2dfd25976..438da2b28 100644 --- a/lib/postzord.rb +++ b/lib/postzord.rb @@ -3,5 +3,6 @@ # the COPYRIGHT file. module Postzord - + require 'postzord/receiver' + require 'postzord/dispatcher' end diff --git a/lib/postzord/dispatcher.rb b/lib/postzord/dispatcher.rb index 651087d5b..ae6fa2687 100644 --- a/lib/postzord/dispatcher.rb +++ b/lib/postzord/dispatcher.rb @@ -4,8 +4,8 @@ class Postzord::Dispatcher - require Rails.root.join('lib', 'postzord', 'dispatcher', 'private') - require Rails.root.join('lib', 'postzord', 'dispatcher', 'public') + require 'postzord/dispatcher/private' + require 'postzord/dispatcher/public' attr_reader :sender, :object, :xml, :subscribers, :opts diff --git a/lib/postzord/receiver.rb b/lib/postzord/receiver.rb index 9b0ad6408..f411de7c3 100644 --- a/lib/postzord/receiver.rb +++ b/lib/postzord/receiver.rb @@ -4,8 +4,9 @@ class Postzord::Receiver - require Rails.root.join('lib', 'postzord', 'receiver', 'private') - require Rails.root.join('lib', 'postzord', 'receiver', 'public') + require 'postzord/receiver/private' + require 'postzord/receiver/public' + require 'postzord/receiver/local_batch' def perform! self.receive! diff --git a/lib/postzord/receiver/private.rb b/lib/postzord/receiver/private.rb index e966dae18..3e9517073 100644 --- a/lib/postzord/receiver/private.rb +++ b/lib/postzord/receiver/private.rb @@ -2,9 +2,6 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -require Rails.root.join('lib', 'webfinger') -require Rails.root.join('lib', 'diaspora', 'parser') - class Postzord::Receiver::Private < Postzord::Receiver def initialize(user, opts={}) diff --git a/lib/salmon/salmon.rb b/lib/salmon.rb similarity index 76% rename from lib/salmon/salmon.rb rename to lib/salmon.rb index c3ce02ae6..747190d60 100644 --- a/lib/salmon/salmon.rb +++ b/lib/salmon.rb @@ -24,7 +24,7 @@ end # Verify documents secured with Magic Signatures module Salmon - autoload :Slap, File.join(Rails.root, "lib", "salmon", "slap").to_s - autoload :EncryptedSlap, File.join(Rails.root, "lib", "salmon", "encrypted_slap").to_s - autoload :MagicSigEnvelope, File.join(Rails.root, "lib", "salmon", "magic_sig_envelope").to_s + require "salmon/slap" + require "salmon/encrypted_slap" + require "salmon/magic_sig_envelope" end diff --git a/lib/stream.rb b/lib/stream.rb index e778980cf..ae5c98f26 100644 --- a/lib/stream.rb +++ b/lib/stream.rb @@ -1,3 +1,12 @@ module Stream - + require 'stream/activity' + require 'stream/aspect' + require 'stream/comments' + require 'stream/followed_tag' + require 'stream/likes' + require 'stream/mention' + require 'stream/multi' + require 'stream/person' + require 'stream/public' + require 'stream/tag' end diff --git a/lib/stream/base.rb b/lib/stream/base.rb index 47739a6be..61eba6f93 100644 --- a/lib/stream/base.rb +++ b/lib/stream/base.rb @@ -1,4 +1,3 @@ -require Rails.root.join("lib", "publisher") class Stream::Base TYPES_OF_POST_IN_STREAM = ['StatusMessage', 'Reshare', 'ActivityStreams::Photo'] diff --git a/lib/webfinger.rb b/lib/webfinger.rb index d5d50b79f..c70cb7469 100644 --- a/lib/webfinger.rb +++ b/lib/webfinger.rb @@ -2,9 +2,6 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -require Rails.root.join('lib', 'hcard') -require Rails.root.join('lib', 'webfinger_profile') - class Webfinger attr_accessor :host_meta_xrd, :webfinger_profile_xrd, :webfinger_profile, :hcard, :hcard_xrd, :person, diff --git a/spec/lib/diaspora/fetcher/public_spec.rb b/spec/lib/diaspora/fetcher/public_spec.rb index 394b84c87..5056e9c52 100644 --- a/spec/lib/diaspora/fetcher/public_spec.rb +++ b/spec/lib/diaspora/fetcher/public_spec.rb @@ -3,24 +3,26 @@ # the COPYRIGHT file. require 'spec_helper' -require Rails.root.join('lib','diaspora','fetcher','public') # Tests fetching public posts of a person on a remote server -describe PublicFetcher do +describe Diaspora::Fetcher::Public do before do # the fixture is taken from an actual json request. # it contains 10 StatusMessages and 5 Reshares, all of them public # the guid of the person is "7445f9a0a6c28ebb" @fixture = File.open(Rails.root.join('spec', 'fixtures', 'public_posts.json')).read - @fetcher = PublicFetcher.new + @fetcher = Diaspora::Fetcher::Public.new @person = FactoryGirl.create(:person, {:guid => "7445f9a0a6c28ebb", :url => "https://remote-testpod.net", :diaspora_handle => "testuser@remote-testpod.net"}) stub_request(:get, /remote-testpod.net\/people\/.*/) - .with(:headers => {'Accept'=>'application/json'}) - .to_return(:body => @fixture) + .with(headers: { + 'Accept'=>'application/json', + 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'User-Agent'=>'diaspora-fetcher' + }).to_return(:body => @fixture) end describe "#retrieve_posts" do @@ -34,8 +36,8 @@ describe PublicFetcher do it "sets the operation status on the person" do @person.reload - @person.fetch_status.should_not eql(PublicFetcher::Status_Initial) - @person.fetch_status.should eql(PublicFetcher::Status_Fetched) + @person.fetch_status.should_not eql(Diaspora::Fetcher::Public::Status_Initial) + @person.fetch_status.should eql(Diaspora::Fetcher::Public::Status_Fetched) end it "sets the @data variable to the parsed JSON data" do @@ -73,8 +75,8 @@ describe PublicFetcher do } @person.reload - @person.fetch_status.should_not eql(PublicFetcher::Status_Initial) - @person.fetch_status.should eql(PublicFetcher::Status_Processed) + @person.fetch_status.should_not eql(Diaspora::Fetcher::Public::Status_Initial) + @person.fetch_status.should eql(Diaspora::Fetcher::Public::Status_Processed) end context 'created post' do @@ -121,7 +123,7 @@ describe PublicFetcher do end context "private methods" do - let(:public_fetcher) { PublicFetcher.new } + let(:public_fetcher) { Diaspora::Fetcher::Public.new } describe '#qualifies_for_fetching?' do it "raises an error if the person doesn't exist" do @@ -135,7 +137,7 @@ describe PublicFetcher do it 'returns false if the person is unfetchable' do public_fetcher.instance_eval { - @person = FactoryGirl.create(:person, {:fetch_status => PublicFetcher::Status_Unfetchable}) + @person = FactoryGirl.create(:person, {:fetch_status => Diaspora::Fetcher::Public::Status_Unfetchable}) qualifies_for_fetching? }.should be_false end @@ -146,12 +148,12 @@ describe PublicFetcher do @person = user.person qualifies_for_fetching? }.should be_false - user.person.fetch_status.should eql PublicFetcher::Status_Unfetchable + user.person.fetch_status.should eql Diaspora::Fetcher::Public::Status_Unfetchable end it 'returns false if the person is processing already (or has been processed)' do person = FactoryGirl.create(:person) - person.fetch_status = PublicFetcher::Status_Fetched + person.fetch_status = Diaspora::Fetcher::Public::Status_Fetched person.save public_fetcher.instance_eval { @person = person @@ -173,14 +175,14 @@ describe PublicFetcher do person = @person public_fetcher.instance_eval { @person = person - set_fetch_status PublicFetcher::Status_Unfetchable + set_fetch_status Diaspora::Fetcher::Public::Status_Unfetchable } - @person.fetch_status.should eql PublicFetcher::Status_Unfetchable + @person.fetch_status.should eql Diaspora::Fetcher::Public::Status_Unfetchable public_fetcher.instance_eval { - set_fetch_status PublicFetcher::Status_Initial + set_fetch_status Diaspora::Fetcher::Public::Status_Initial } - @person.fetch_status.should eql PublicFetcher::Status_Initial + @person.fetch_status.should eql Diaspora::Fetcher::Public::Status_Initial end end diff --git a/spec/lib/email_inviter_spec.rb b/spec/lib/email_inviter_spec.rb index 3d423d362..2de128287 100644 --- a/spec/lib/email_inviter_spec.rb +++ b/spec/lib/email_inviter_spec.rb @@ -1,4 +1,4 @@ -require File.join(File.dirname(__FILE__), '..', '..', 'lib', 'email_inviter') +require 'spec_helper' describe EmailInviter do before do @@ -53,4 +53,4 @@ describe EmailInviter do inviter.invitation_code.should == @user.invitation_code end end -end \ No newline at end of file +end diff --git a/spec/lib/hcard_spec.rb b/spec/lib/hcard_spec.rb index 4ab77030a..4a898d9fb 100644 --- a/spec/lib/hcard_spec.rb +++ b/spec/lib/hcard_spec.rb @@ -3,7 +3,6 @@ # the COPYRIGHT file. require 'spec_helper' -require Rails.root.join('lib', 'hcard') describe HCard do it 'should parse an hcard' do diff --git a/spec/lib/postzord/dispatcher/private_spec.rb b/spec/lib/postzord/dispatcher/private_spec.rb index e09f12819..84b697695 100644 --- a/spec/lib/postzord/dispatcher/private_spec.rb +++ b/spec/lib/postzord/dispatcher/private_spec.rb @@ -3,7 +3,6 @@ # the COPYRIGHT file. require 'spec_helper' -require Rails.root.join('lib', 'postzord', 'dispatcher', 'private') describe Postzord::Dispatcher::Private do diff --git a/spec/lib/postzord/dispatcher_spec.rb b/spec/lib/postzord/dispatcher_spec.rb index 7df98d97b..4e39a1d95 100644 --- a/spec/lib/postzord/dispatcher_spec.rb +++ b/spec/lib/postzord/dispatcher_spec.rb @@ -4,8 +4,6 @@ require 'spec_helper' -require Rails.root.join('lib', 'postzord', 'dispatcher') - describe Postzord::Dispatcher do before do @sm = FactoryGirl.create(:status_message, :public => true, :author => alice.person) diff --git a/spec/lib/postzord/receiver/local_batch_spec.rb b/spec/lib/postzord/receiver/local_batch_spec.rb index 59867d710..ad1ef7115 100644 --- a/spec/lib/postzord/receiver/local_batch_spec.rb +++ b/spec/lib/postzord/receiver/local_batch_spec.rb @@ -1,5 +1,4 @@ require 'spec_helper' -require Rails.root.join('lib', 'postzord', 'receiver', 'local_batch') describe Postzord::Receiver::LocalBatch do before do diff --git a/spec/lib/postzord/receiver/private_spec.rb b/spec/lib/postzord/receiver/private_spec.rb index 237346976..4b2bb653c 100644 --- a/spec/lib/postzord/receiver/private_spec.rb +++ b/spec/lib/postzord/receiver/private_spec.rb @@ -4,9 +4,6 @@ require 'spec_helper' -require Rails.root.join('lib', 'postzord') -require Rails.root.join('lib', 'postzord', 'receiver', 'private') - describe Postzord::Receiver::Private do before do diff --git a/spec/lib/postzord/receiver/public_spec.rb b/spec/lib/postzord/receiver/public_spec.rb index 188586416..766686677 100644 --- a/spec/lib/postzord/receiver/public_spec.rb +++ b/spec/lib/postzord/receiver/public_spec.rb @@ -4,9 +4,6 @@ require 'spec_helper' -require Rails.root.join('lib', 'postzord') -require Rails.root.join('lib', 'postzord', 'receiver', 'public') - describe Postzord::Receiver::Public do before do @post = FactoryGirl.build(:status_message, :author => alice.person, :public => true) diff --git a/spec/lib/postzord/receiver_spec.rb b/spec/lib/postzord/receiver_spec.rb index 8eb2a55bc..5ac6dbf9a 100644 --- a/spec/lib/postzord/receiver_spec.rb +++ b/spec/lib/postzord/receiver_spec.rb @@ -4,8 +4,6 @@ require 'spec_helper' -require Rails.root.join('lib', 'postzord', 'receiver') - describe Postzord::Receiver do before do @receiver = Postzord::Receiver.new diff --git a/spec/lib/publisher_spec.rb b/spec/lib/publisher_spec.rb index 129d47983..c638235ff 100644 --- a/spec/lib/publisher_spec.rb +++ b/spec/lib/publisher_spec.rb @@ -1,9 +1,6 @@ require 'spec_helper' -#NOTE;why is it not auto loadeded? -require Rails.root.join('lib', 'publisher') - describe Publisher do before do @publisher = Publisher.new(alice) diff --git a/spec/lib/pubsubhubbub_spec.rb b/spec/lib/pubsubhubbub_spec.rb index bcbf8431a..c486f0377 100644 --- a/spec/lib/pubsubhubbub_spec.rb +++ b/spec/lib/pubsubhubbub_spec.rb @@ -3,7 +3,6 @@ # the COPYRIGHT file. require 'spec_helper' -require Rails.root.join('lib', 'pubsubhubbub') describe Pubsubhubbub do describe '#publish' do @@ -16,4 +15,4 @@ describe Pubsubhubbub do Pubsubhubbub.new(hub).publish(feed).should be_success end end -end \ No newline at end of file +end diff --git a/spec/lib/rake_helper_spec.rb b/spec/lib/rake_helper_spec.rb index 77fd6e198..58039173c 100644 --- a/spec/lib/rake_helper_spec.rb +++ b/spec/lib/rake_helper_spec.rb @@ -3,7 +3,8 @@ # the COPYRIGHT file. require 'spec_helper' -require Rails.root.join('lib', 'rake_helpers') +require 'rake_helpers' + include RakeHelpers describe RakeHelpers do before do diff --git a/spec/lib/statistics_spec.rb b/spec/lib/statistics_spec.rb index f626ccd57..9720d2631 100644 --- a/spec/lib/statistics_spec.rb +++ b/spec/lib/statistics_spec.rb @@ -1,5 +1,4 @@ require 'spec_helper' -require Rails.root.join('lib', 'statistics') describe Statistics do diff --git a/spec/lib/stream/base_spec.rb b/spec/lib/stream/base_spec.rb index e89b3dadf..104e3699d 100644 --- a/spec/lib/stream/base_spec.rb +++ b/spec/lib/stream/base_spec.rb @@ -1,5 +1,6 @@ require 'spec_helper' require Rails.root.join('spec', 'shared_behaviors', 'stream') + describe Stream::Base do before do @stream = Stream::Base.new(alice) diff --git a/spec/lib/template_picker_spec.rb b/spec/lib/template_picker_spec.rb index 1e1570910..119ce3ac7 100644 --- a/spec/lib/template_picker_spec.rb +++ b/spec/lib/template_picker_spec.rb @@ -1,5 +1,5 @@ -require File.join(File.dirname(__FILE__), '..', '..', 'lib', 'template_picker') require 'spec_helper' + describe TemplatePicker do before do @post_stubs = {:type => 'StatusMessage', :photos => stub(:size => 2), diff --git a/spec/models/jobs/publish_to_hub_spec.rb b/spec/models/jobs/publish_to_hub_spec.rb index 38e5e622c..27937cca5 100644 --- a/spec/models/jobs/publish_to_hub_spec.rb +++ b/spec/models/jobs/publish_to_hub_spec.rb @@ -3,7 +3,6 @@ # the COPYRIGHT file. require 'spec_helper' -require Rails.root.join('lib', 'pubsubhubbub') describe Jobs::PublishToHub do describe '.perform' do From 79a79d65d684b6cfa93e1d2dde9ac7abcfb7a481 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonne=20Ha=C3=9F?= Date: Tue, 19 Feb 2013 09:34:41 +0100 Subject: [PATCH 2/2] Bye Resque. Ohai Sidekiq. * Dropped all references to Resque * Moved all jobs under app/workers since that's the Sidekiq convention * Renamed Jobs module to Worker to match new location * Adapted all jobs to Sidekiq * Replaced all enqueue calls with perform_async * Dropped Resque hacks from specs and features, replaced with sidekig/testing in RSpec and sidekig/testing/inline in Cucumber * Updated scripts to start a Sidekiq server * Inline Sidekiq sinatra app * Let Sidekiq create the actual Redis instance * Workaround already initialized constant warnings in service models * Resolved ToDo in one job definition by creating proper exception clases for some errors in receiving posts * Added sidekiq section to configuration to make it completly configurable to the user * Add Sidekiq middleware for clean backtraces * Delay HttpMulti retry to give offline pods a chance to come back up * Do not retry on GUID already taken and alike errors * Be graceful about deleted posts in GatherOEmbedData --- .foreman | 2 +- Changelog.md | 106 ++++++++++++++++-- Gemfile | 5 +- Gemfile.lock | 30 +++-- Procfile | 2 +- Rakefile | 1 - app/controllers/publics_controller.rb | 4 +- app/controllers/services_controller.rb | 2 +- app/models/account_deletion.rb | 2 +- app/models/jobs/base.rb | 25 ----- app/models/jobs/gather_o_embed_data.rb | 16 --- app/models/jobs/receive.rb | 17 --- app/models/jobs/receive_unencrypted_salmon.rb | 21 ---- app/models/notifications/also_commented.rb | 2 +- app/models/notifications/comment_on_post.rb | 2 +- app/models/notifications/liked.rb | 2 +- app/models/notifications/mentioned.rb | 2 +- app/models/notifications/private_message.rb | 2 +- app/models/notifications/request_accepted.rb | 2 +- app/models/notifications/reshared.rb | 3 +- app/models/notifications/started_sharing.rb | 2 +- app/models/photo.rb | 2 +- app/models/status_message.rb | 2 +- app/models/user.rb | 8 +- app/views/admins/_admin_bar.haml | 3 +- app/workers/base.rb | 23 ++++ .../jobs => workers}/deferred_dispatch.rb | 6 +- .../jobs => workers}/delete_account.rb | 7 +- .../delete_post_from_service.rb | 6 +- .../jobs => workers}/fetch_profile_photo.rb | 7 +- .../jobs => workers}/fetch_public_posts.rb | 10 +- .../jobs => workers}/fetch_webfinger.rb | 8 +- app/workers/gather_o_embed_data.rb | 22 ++++ app/{models/jobs => workers}/http_multi.rb | 14 +-- .../jobs => workers}/mail/also_commented.rb | 7 +- .../jobs => workers}/mail/comment_on_post.rb | 7 +- .../jobs => workers}/mail/confirm_email.rb | 7 +- .../mail/invite_user_by_email.rb | 8 +- app/{models/jobs => workers}/mail/liked.rb | 7 +- .../jobs => workers}/mail/mentioned.rb | 7 +- .../jobs => workers}/mail/private_message.rb | 7 +- app/{models/jobs => workers}/mail/reshared.rb | 7 +- .../jobs => workers}/mail/started_sharing.rb | 7 +- .../jobs => workers}/notify_local_users.rb | 8 +- .../jobs => workers}/post_to_service.rb | 6 +- app/{models/jobs => workers}/process_photo.rb | 7 +- .../jobs => workers}/publish_to_hub.rb | 7 +- app/workers/receive.rb | 19 ++++ .../receive_encrypted_salmon.rb | 7 +- .../jobs => workers}/receive_local_batch.rb | 10 +- app/workers/receive_unencrypted_salmon.rb | 21 ++++ .../jobs => workers}/resend_invitation.rb | 7 +- .../jobs => workers}/reset_password.rb | 6 +- config/database.yml.example | 19 ++-- config/defaults.yml | 13 ++- config/diaspora.yml.example | 44 ++++++-- config/initializers/load_mail_jobs.rb | 1 - config/initializers/resque.rb | 36 ------ config/initializers/sidekiq.rb | 50 +++++++++ config/locales/diaspora/en.yml | 2 +- config/routes.rb | 11 +- config/unicorn.rb | 17 ++- db/seeds.rb | 2 +- features/support/env.rb | 6 +- features/support/poor_mans_webmock.rb | 16 +-- features/support/run_resque_in_process.rb | 5 - lib/admin_rack.rb | 19 ---- lib/configuration_methods.rb | 16 ++- lib/diaspora/exceptions.rb | 13 +++ lib/diaspora/fetcher/public.rb | 2 +- lib/diaspora/parser.rb | 2 +- lib/postzord/dispatcher.rb | 27 ++--- lib/postzord/receiver/private.rb | 6 +- lib/postzord/receiver/public.rb | 4 +- lib/sidekiq_middlewares.rb | 13 +++ lib/tasks/resque.rake | 18 --- lib/webfinger.rb | 2 +- script/ci/build.sh | 2 +- script/server | 15 +-- .../devise/passwords_controller_spec.rb | 9 +- spec/controllers/publics_controller_spec.rb | 6 +- spec/controllers/services_controller_spec.rb | 4 +- .../status_messages_controller_spec.rb | 7 +- spec/controllers/users_controller_spec.rb | 10 +- spec/integration/attack_vectors_spec.rb | 26 ++--- spec/integration/dispatching_spec.rb | 7 +- spec/integration/receiving_spec.rb | 15 ++- spec/lib/configuration_methods_spec.rb | 41 ++++--- spec/lib/postzord/dispatcher_spec.rb | 28 ++--- spec/lib/postzord/receiver/public_spec.rb | 6 +- spec/lib/webfinger_spec.rb | 4 +- spec/models/account_deletion_spec.rb | 4 +- spec/models/photo_spec.rb | 6 +- spec/models/status_message_spec.rb | 2 +- spec/models/user_spec.rb | 20 ++-- spec/spec_helper.rb | 1 + spec/support/fake_resque.rb | 19 ---- spec/support/fake_typhoeus.rb | 8 +- spec/support/inlined_jobs.rb | 10 ++ spec/support/user_methods.rb | 4 +- .../jobs => workers}/delete_account_spec.rb | 4 +- .../delete_post_from_service_spec.rb | 4 +- .../fetch_profile_photo_spec.rb | 10 +- .../gather_o_embed_data_spec.rb | 18 ++- .../jobs => workers}/http_multi_spec.rb | 20 ++-- .../mail/invite_user_by_email_spec.rb | 4 +- .../jobs => workers}/mail/mentioned_spec.rb | 4 +- .../mail/private_message_spec.rb | 4 +- .../jobs => workers}/mail/reshared_spec.rb | 4 +- .../notify_local_users_spec.rb | 4 +- .../jobs => workers}/post_to_service_spec.rb | 4 +- .../jobs => workers}/process_photo_spec.rb | 16 +-- .../jobs => workers}/publish_to_hub_spec.rb | 4 +- .../jobs => workers}/receive_salmon_spec.rb | 4 +- spec/{models/jobs => workers}/receive_spec.rb | 4 +- .../resend_invitation_spec.rb | 4 +- .../jobs => workers}/reset_password_spec.rb | 4 +- 117 files changed, 688 insertions(+), 556 deletions(-) delete mode 100644 app/models/jobs/base.rb delete mode 100644 app/models/jobs/gather_o_embed_data.rb delete mode 100644 app/models/jobs/receive.rb delete mode 100644 app/models/jobs/receive_unencrypted_salmon.rb create mode 100644 app/workers/base.rb rename app/{models/jobs => workers}/deferred_dispatch.rb (85%) rename app/{models/jobs => workers}/delete_account.rb (75%) rename app/{models/jobs => workers}/delete_post_from_service.rb (75%) rename app/{models/jobs => workers}/fetch_profile_photo.rb (87%) rename app/{models/jobs => workers}/fetch_public_posts.rb (53%) rename app/{models/jobs => workers}/fetch_webfinger.rb (64%) create mode 100644 app/workers/gather_o_embed_data.rb rename app/{models/jobs => workers}/http_multi.rb (60%) rename app/{models/jobs => workers}/mail/also_commented.rb (64%) rename app/{models/jobs => workers}/mail/comment_on_post.rb (58%) rename app/{models/jobs => workers}/mail/confirm_email.rb (58%) rename app/{models/jobs => workers}/mail/invite_user_by_email.rb (80%) rename app/{models/jobs => workers}/mail/liked.rb (54%) rename app/{models/jobs => workers}/mail/mentioned.rb (73%) rename app/{models/jobs => workers}/mail/private_message.rb (73%) rename app/{models/jobs => workers}/mail/reshared.rb (55%) rename app/{models/jobs => workers}/mail/started_sharing.rb (73%) rename app/{models/jobs => workers}/notify_local_users.rb (79%) rename app/{models/jobs => workers}/post_to_service.rb (76%) rename app/{models/jobs => workers}/process_photo.rb (86%) rename app/{models/jobs => workers}/publish_to_hub.rb (71%) create mode 100644 app/workers/receive.rb rename app/{models/jobs => workers}/receive_encrypted_salmon.rb (73%) rename app/{models/jobs => workers}/receive_local_batch.rb (60%) create mode 100644 app/workers/receive_unencrypted_salmon.rb rename app/{models/jobs => workers}/resend_invitation.rb (76%) rename app/{models/jobs => workers}/reset_password.rb (66%) delete mode 100644 config/initializers/load_mail_jobs.rb delete mode 100644 config/initializers/resque.rb create mode 100644 config/initializers/sidekiq.rb delete mode 100644 features/support/run_resque_in_process.rb delete mode 100644 lib/admin_rack.rb create mode 100644 lib/sidekiq_middlewares.rb delete mode 100644 lib/tasks/resque.rake delete mode 100644 spec/support/fake_resque.rb create mode 100644 spec/support/inlined_jobs.rb rename spec/{models/jobs => workers}/delete_account_spec.rb (84%) rename spec/{models/jobs => workers}/delete_post_from_service_spec.rb (76%) rename spec/{models/jobs => workers}/fetch_profile_photo_spec.rb (84%) rename spec/{models/jobs => workers}/gather_o_embed_data_spec.rb (75%) rename spec/{models/jobs => workers}/http_multi_spec.rb (77%) rename spec/{models/jobs => workers}/mail/invite_user_by_email_spec.rb (80%) rename spec/{models/jobs => workers}/mail/mentioned_spec.rb (83%) rename spec/{models/jobs => workers}/mail/private_message_spec.rb (87%) rename spec/{models/jobs => workers}/mail/reshared_spec.rb (84%) rename spec/{models/jobs => workers}/notify_local_users_spec.rb (82%) rename spec/{models/jobs => workers}/post_to_service_spec.rb (80%) rename spec/{models/jobs => workers}/process_photo_spec.rb (71%) rename spec/{models/jobs => workers}/publish_to_hub_spec.rb (85%) rename spec/{models/jobs => workers}/receive_salmon_spec.rb (78%) rename spec/{models/jobs => workers}/receive_spec.rb (83%) rename spec/{models/jobs => workers}/resend_invitation_spec.rb (82%) rename spec/{models/jobs => workers}/reset_password_spec.rb (79%) diff --git a/.foreman b/.foreman index da229edb6..a39123f3a 100644 --- a/.foreman +++ b/.foreman @@ -1,2 +1,2 @@ port: 3000 -formation: web=1,worker=0 +formation: web=1,sidekiq=0 diff --git a/Changelog.md b/Changelog.md index 727e72c6d..43c8e0f35 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,102 @@ # Head +## Refactor + +### Replaced Resque with Sidekiq - Migration guide - [#3993](https://github.com/diaspora/diaspora/pull/3993) + +We replaced our queue system with Sidekiq. You might know that Resque needs Redis. +Sidekiq does too, so don't remove it, it's still required. Sidekiq uses a threaded +model so you'll need far less processes than with Resque to do the same amount +of work. + +To update do the following: + +1. Before updating (even before the `git pull`!) stop your application + server (Unicorn by default, started through Foreman). +2. In case you did already run `git pull` checkout v0.0.3.2: + + ``` + git fetch origin + git checkout v0.0.3.2 + bundle + ``` + +3. Start Resque web (you'll need temporary access to port 5678, check + your Firewall if needed!): + + ``` + bundle exec resque-web + ``` + + In case you need it you can adjust the port with the `-p` flag. +4. One last time, start a Resque worker: + + ``` + RAILS_ENV=production QUEUE=* bundle exec rake resque:work + ``` + + Visit Resque web via http://your_host:5678, wait until all queues but the + failed one are empty (show 0 jobs). +5. Kill the Resque worker by hitting Ctrl+C. Kill Resque web with: + + ``` + bundle exec resque-web -k + ``` + + Don't forget to close the port on the Firewall again, if you had to open it. +6. In case you needed to do step 2., run: + + ``` + git checkout master + bundle + ``` + +7. Proceed with the update as normal (migrate database, precompile assets). +8. Before starting Diaspora again ensure that you reviewed the new + `environment.sidekiq` section in `config/diaspora.yml.example` and, + if wanted, transfered it to your `config/diaspora.yml` and made any + needed changes. In particular increase the `environment.sidekiq.concurrency` + setting on any medium sized pod. If you do change that value, edit + your `config/database.yml` and add a matching `pool: n` to your database + configuration. n should be equal or higher than the amount of + threads per Sidekiq worker. This sets how many concurrent + connections to the database ActiveRecord allows. + + +If you aren't using `script/server` but for example passenger, you no +longer need to start a Resque worker, but a Sidekiq worker now. The +command for that is: + +``` +bundle exec sidekiq +``` + + +#### Heroku + +The only gotcha for Heroku single gear setups is that the setting name +to spawn a background worker from the unicorn process changed. Run + +``` +heroku config:remove SERVER_EMBED_RESQUE_WORKER +heroku config:set SERVER_EMBED_SIDEKIQ_WORKER=true +``` + +We're automatically adjusting the ActiveRecord connection pool size for you. + +Larger Heroku setups should have enough expertise to figure out what to do +by them self. + +### Other + +* Cleaned up requires of our own libraries [#3993](https://github.com/diaspora/diaspora/pull/3993) +* Refactor people_controller#show and photos_controller#index [#4002](https://github.com/diaspora/diaspora/issues/4002) +* Modularize layout [#3944](https://github.com/diaspora/diaspora/pull/3944) +* Add header to the sign up page [#3944](https://github.com/diaspora/diaspora/pull/3944) +* Add a configuration entry to set max-age header to Amazon S3 resources. [#4048](https://github.com/diaspora/diaspora/pull/4048) +* Load images via sprites [#4039](https://github.com/diaspora/diaspora/pull/4039) +* Delete unnecessary javascript views. [#4059](https://github.com/diaspora/diaspora/pull/4059) + ## Bug fixes * reset comment box height after posting a comment. [#4030](https://github.com/diaspora/diaspora/issues/4030) @@ -13,15 +110,6 @@ * Fix mobile view of deleted reshares. [#4063](https://github.com/diaspora/diaspora/issues/4063) * Hide comment button in the mobile view when not signed in. [#4065](https://github.com/diaspora/diaspora/issues/4065) -## Refactor - -* Delete unnecessary javascript views. [#4059] (https://github.com/diaspora/diaspora/pull/4059) -* Add a configuration entry to set max-age header to Amazon S3 resources. [#4048](https://github.com/diaspora/diaspora/pull/4048) -* Refactor people_controller#show and photos_controller#index [#4002](https://github.com/diaspora/diaspora/issues/4002) -* Modularize layout [#3944](https://github.com/diaspora/diaspora/pull/3944) -* Add header to the sign up page [#3944](https://github.com/diaspora/diaspora/pull/3944) -* Load images via sprites [#4039](https://github.com/diaspora/diaspora/pull/4039) - ## Features * Deleting a post that was shared to Facebook now deletes it from Facebook too [#3980]( https://github.com/diaspora/diaspora/pull/3980) diff --git a/Gemfile b/Gemfile index f5aeab9e4..44e63c489 100644 --- a/Gemfile +++ b/Gemfile @@ -17,8 +17,9 @@ gem 'devise', '2.1.3' # Background processing -gem 'resque', '1.23.0' -gem 'resque-timeout', '1.0.0' +gem 'sidekiq', '2.7.5' +gem 'sinatra', '1.3.3' +gem 'slim', '1.3.6' # Configuration diff --git a/Gemfile.lock b/Gemfile.lock index 2729dde58..100d7845c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -62,6 +62,9 @@ GEM carrierwave (0.8.0) activemodel (>= 3.2.0) activesupport (>= 3.2.0) + celluloid (0.12.4) + facter (>= 1.6.12) + timers (>= 1.0.0) childprocess (0.3.9) ffi (~> 1.0, >= 1.0.11) chunky_png (1.2.7) @@ -81,6 +84,7 @@ GEM compass-rails (1.0.3) compass (>= 0.12.2, < 0.14) configurate (0.0.2) + connection_pool (1.0.0) crack (0.3.2) cucumber (1.2.3) builder (>= 2.1.2) @@ -103,6 +107,7 @@ GEM excon (0.20.1) execjs (1.4.0) multi_json (~> 1.0) + facter (1.6.17) factory_girl (4.2.0) activesupport (>= 3.0.0) factory_girl_rails (4.2.1) @@ -321,13 +326,6 @@ GEM redis-namespace (1.2.1) redis (~> 3.0.0) remotipart (1.0.5) - resque (1.23.0) - multi_json (~> 1.0) - redis-namespace (~> 1.0) - sinatra (>= 0.9.2) - vegas (~> 0.1.2) - resque-timeout (1.0.0) - resque (~> 1.0) rmagick (2.13.2) roxml (3.1.6) activesupport (>= 2.3.0) @@ -363,11 +361,20 @@ GEM multi_json (~> 1.0) rubyzip websocket (~> 1.0.4) + sidekiq (2.7.5) + celluloid (~> 0.12.0) + connection_pool (~> 1.0) + multi_json (~> 1) + redis (~> 3) + redis-namespace simple_oauth (0.2.0) sinatra (1.3.3) rack (~> 1.3, >= 1.3.6) rack-protection (~> 1.2) tilt (~> 1.3, >= 1.3.3) + slim (1.3.6) + temple (~> 0.5.5) + tilt (~> 1.3.3) slop (3.4.4) spork (1.0.0rc3) sprockets (2.2.2) @@ -376,10 +383,12 @@ GEM rack (~> 1.0) tilt (~> 1.1, != 1.3.0) subexec (0.2.2) + temple (0.5.5) terminal-table (1.4.5) thor (0.17.0) tilt (1.3.6) timecop (0.6.1) + timers (1.1.0) treetop (1.4.12) polyglot polyglot (>= 0.3.1) @@ -397,8 +406,6 @@ GEM kgio (~> 2.6) rack raindrops (~> 0.7) - vegas (0.1.11) - rack (>= 1.0.0) warden (1.2.1) rack (>= 1.0) webmock (1.8.11) @@ -473,8 +480,6 @@ DEPENDENCIES rb-inotify (= 0.9.0) redcarpet (= 2.2.2) remotipart (= 1.0.5) - resque (= 1.23.0) - resque-timeout (= 1.0.0) rmagick (= 2.13.2) roxml (= 3.1.6) rspec-instafail (= 0.2.4) @@ -482,6 +487,9 @@ DEPENDENCIES ruby-oembed (= 0.8.8) sass-rails (= 3.2.6) selenium-webdriver (= 2.31.0) + sidekiq (= 2.7.5) + sinatra (= 1.3.3) + slim (= 1.3.6) spork (= 1.0.0rc3) timecop (= 0.6.1) twitter (= 4.6.2) diff --git a/Procfile b/Procfile index 999e5c582..109ee2337 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,2 @@ web: bundle exec unicorn_rails -c config/unicorn.rb -p $PORT -worker: env QUEUE=* bundle exec rake resque:work +sidekiq: bundle exec sidekiq diff --git a/Rakefile b/Rakefile index 660a61ee1..2c022e0b0 100644 --- a/Rakefile +++ b/Rakefile @@ -7,7 +7,6 @@ require File.expand_path('../config/application', __FILE__) require 'rake' -require 'resque/tasks' # for rake 0.9.0 module Diaspora diff --git a/app/controllers/publics_controller.rb b/app/controllers/publics_controller.rb index 646e68480..c0724190e 100644 --- a/app/controllers/publics_controller.rb +++ b/app/controllers/publics_controller.rb @@ -49,7 +49,7 @@ class PublicsController < ApplicationController def receive_public FEDERATION_LOGGER.info("recieved a public message") - Resque.enqueue(Jobs::ReceiveUnencryptedSalmon, CGI::unescape(params[:xml])) + Workers::ReceiveUnencryptedSalmon.perform_async(CGI::unescape(params[:xml])) render :nothing => true, :status => :ok end @@ -65,7 +65,7 @@ class PublicsController < ApplicationController @user = person.owner FEDERATION_LOGGER.info("recieved a private message for user:#{@user.id}") - Resque.enqueue(Jobs::ReceiveEncryptedSalmon, @user.id, CGI::unescape(params[:xml])) + Workers::ReceiveEncryptedSalmon.perform_async(@user.id, CGI::unescape(params[:xml])) render :nothing => true, :status => 202 end diff --git a/app/controllers/services_controller.rb b/app/controllers/services_controller.rb index 2ff6c3717..6ded9b740 100644 --- a/app/controllers/services_controller.rb +++ b/app/controllers/services_controller.rb @@ -36,7 +36,7 @@ class ServicesController < ApplicationController fetch_photo = current_user.profile[:image_url].blank? current_user.update_profile(current_user.profile.from_omniauth_hash(user)) - Resque.enqueue(Jobs::FetchProfilePhoto, current_user.id, service.id, user["image"]) if fetch_photo + Workers::FetchProfilePhoto.perform_async(current_user.id, service.id, user["image"]) if fetch_photo flash[:notice] = I18n.t 'services.create.success' else diff --git a/app/models/account_deletion.rb b/app/models/account_deletion.rb index 581593b04..c3ca2e142 100644 --- a/app/models/account_deletion.rb +++ b/app/models/account_deletion.rb @@ -26,7 +26,7 @@ class AccountDeletion < ActiveRecord::Base end def queue_delete_account - Resque.enqueue(Jobs::DeleteAccount, self.id) + Workers::DeleteAccount.perform_async(self.id) end def perform! diff --git a/app/models/jobs/base.rb b/app/models/jobs/base.rb deleted file mode 100644 index 4c4d29416..000000000 --- a/app/models/jobs/base.rb +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) 2010-2011, Diaspora Inc. This file is -# licensed under the Affero General Public License version 3 or later. See -# the COPYRIGHT file. - -module Jobs - class Base - Dir[Rails.root.join('app', 'models', 'jobs', 'mail', '*.rb')].each {|file| require file } - - #TODO these should be subclassed real exceptions - DUMB_ERROR_MESSAGES = [ - "Contact required unless request", - "Relayable object, but no parent object found" ] - - def self.suppress_annoying_errors(&block) - begin - yield - rescue => e - Rails.logger.info("error in job: #{e.message}") - unless DUMB_ERROR_MESSAGES.include?(e.message) - raise e - end - end - end - end -end \ No newline at end of file diff --git a/app/models/jobs/gather_o_embed_data.rb b/app/models/jobs/gather_o_embed_data.rb deleted file mode 100644 index 72fe83d29..000000000 --- a/app/models/jobs/gather_o_embed_data.rb +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2010-2011, Diaspora Inc. This file is -# licensed under the Affero General Public License version 3 or later. See -# the COPYRIGHT file. -# - -module Jobs - class GatherOEmbedData < Base - @queue = :http_service - - def self.perform(post_id, url) - post = Post.find(post_id) - post.o_embed_cache = OEmbedCache.find_or_create_by_url(url) - post.save - end - end -end diff --git a/app/models/jobs/receive.rb b/app/models/jobs/receive.rb deleted file mode 100644 index 04c29d2af..000000000 --- a/app/models/jobs/receive.rb +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) 2010-2011, Diaspora Inc. This file is -# licensed under the Affero General Public License version 3 or later. See -# the COPYRIGHT file. - - -module Jobs - class Receive < Base - - @queue = :receive - def self.perform(user_id, xml, salmon_author_id) - user = User.find(user_id) - salmon_author = Person.find(salmon_author_id) - zord = Postzord::Receiver::Private.new(user, :person => salmon_author) - zord.parse_and_receive(xml) - end - end -end diff --git a/app/models/jobs/receive_unencrypted_salmon.rb b/app/models/jobs/receive_unencrypted_salmon.rb deleted file mode 100644 index 19d33c572..000000000 --- a/app/models/jobs/receive_unencrypted_salmon.rb +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2010-2011, Diaspora Inc. This file is -# licensed under the Affero General Public License version 3 or later. See -# the COPYRIGHT file. - -require Rails.root.join('lib', 'postzord', 'receiver', 'public') - -module Jobs - class ReceiveUnencryptedSalmon < Base - @queue = :receive - - def self.perform(xml) - begin - receiver = Postzord::Receiver::Public.new(xml) - receiver.perform! - rescue => e - FEDERATION_LOGGER.info(e.message) - raise e - end - end - end -end diff --git a/app/models/notifications/also_commented.rb b/app/models/notifications/also_commented.rb index 0268281cf..7b698cd36 100644 --- a/app/models/notifications/also_commented.rb +++ b/app/models/notifications/also_commented.rb @@ -1,6 +1,6 @@ class Notifications::AlsoCommented < Notification def mail_job - Jobs::Mail::AlsoCommented + Workers::Mail::AlsoCommented end def popup_translation_key diff --git a/app/models/notifications/comment_on_post.rb b/app/models/notifications/comment_on_post.rb index f8b6fe45e..631317880 100644 --- a/app/models/notifications/comment_on_post.rb +++ b/app/models/notifications/comment_on_post.rb @@ -1,6 +1,6 @@ class Notifications::CommentOnPost < Notification def mail_job - Jobs::Mail::CommentOnPost + Workers::Mail::CommentOnPost end def popup_translation_key diff --git a/app/models/notifications/liked.rb b/app/models/notifications/liked.rb index abf62f58f..05607b100 100644 --- a/app/models/notifications/liked.rb +++ b/app/models/notifications/liked.rb @@ -1,6 +1,6 @@ class Notifications::Liked < Notification def mail_job - Jobs::Mail::Liked + Workers::Mail::Liked end def popup_translation_key diff --git a/app/models/notifications/mentioned.rb b/app/models/notifications/mentioned.rb index 165395512..16482ace7 100644 --- a/app/models/notifications/mentioned.rb +++ b/app/models/notifications/mentioned.rb @@ -1,6 +1,6 @@ class Notifications::Mentioned < Notification def mail_job - Jobs::Mail::Mentioned + Workers::Mail::Mentioned end def popup_translation_key diff --git a/app/models/notifications/private_message.rb b/app/models/notifications/private_message.rb index 27f0197d9..4ef273041 100644 --- a/app/models/notifications/private_message.rb +++ b/app/models/notifications/private_message.rb @@ -1,6 +1,6 @@ class Notifications::PrivateMessage < Notification def mail_job - Jobs::Mail::PrivateMessage + Workers::Mail::PrivateMessage end def popup_translation_key 'notifications.private_message' diff --git a/app/models/notifications/request_accepted.rb b/app/models/notifications/request_accepted.rb index 4f85e66e0..3651a32f0 100644 --- a/app/models/notifications/request_accepted.rb +++ b/app/models/notifications/request_accepted.rb @@ -1,6 +1,6 @@ class Notifications::RequestAccepted < Notification def mail_job - Jobs::Mail::RequestAcceptance + Workers::Mail::RequestAcceptance end def popup_translation_key 'notifications.request_accepted' diff --git a/app/models/notifications/reshared.rb b/app/models/notifications/reshared.rb index 3032a6ac7..fb9559d6f 100644 --- a/app/models/notifications/reshared.rb +++ b/app/models/notifications/reshared.rb @@ -1,7 +1,6 @@ class Notifications::Reshared < Notification def mail_job - Jobs::Mail::Reshared - #Jobs::Mail::Liked + Workers::Mail::Reshared end def popup_translation_key diff --git a/app/models/notifications/started_sharing.rb b/app/models/notifications/started_sharing.rb index 0d366fe96..ff4975be5 100644 --- a/app/models/notifications/started_sharing.rb +++ b/app/models/notifications/started_sharing.rb @@ -1,6 +1,6 @@ class Notifications::StartedSharing < Notification def mail_job - Jobs::Mail::StartedSharing + Workers::Mail::StartedSharing end def popup_translation_key diff --git a/app/models/photo.rb b/app/models/photo.rb index fdcabedcc..7df5ab9bb 100644 --- a/app/models/photo.rb +++ b/app/models/photo.rb @@ -127,7 +127,7 @@ class Photo < ActiveRecord::Base end def queue_processing_job - Resque.enqueue(Jobs::ProcessPhoto, self.id) + Workers::ProcessPhoto.perform_async(self.id) end def mutable? diff --git a/app/models/status_message.rb b/app/models/status_message.rb index 03f001643..8b0f81c34 100644 --- a/app/models/status_message.rb +++ b/app/models/status_message.rb @@ -155,7 +155,7 @@ class StatusMessage < Post end def queue_gather_oembed_data - Resque.enqueue(Jobs::GatherOEmbedData, self.id, self.oembed_url) + Workers::GatherOEmbedData.perform_async(self.id, self.oembed_url) end def contains_oembed_url_in_text? diff --git a/app/models/user.rb b/app/models/user.rb index 32656c0ed..b16b8b18f 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -164,7 +164,7 @@ class User < ActiveRecord::Base def send_reset_password_instructions generate_reset_password_token! if should_generate_reset_token? - Resque.enqueue(Jobs::ResetPassword, self.id) + Workers::ResetPassword.perform_async(self.id) end def update_user_preferences(pref_hash) @@ -299,15 +299,15 @@ class User < ActiveRecord::Base ######### Mailer ####################### def mail(job, *args) - pref = job.to_s.gsub('Jobs::Mail::', '').underscore + pref = job.to_s.gsub('Workers::Mail::', '').underscore if(self.disable_mail == false && !self.user_preferences.exists?(:email_type => pref)) - Resque.enqueue(job, *args) + job.perform_async(*args) end end def mail_confirm_email return false if unconfirmed_email.blank? - Resque.enqueue(Jobs::Mail::ConfirmEmail, id) + Workers::Mail::ConfirmEmail.perform_async(id) true end diff --git a/app/views/admins/_admin_bar.haml b/app/views/admins/_admin_bar.haml index 0f2577dc7..58fc53070 100644 --- a/app/views/admins/_admin_bar.haml +++ b/app/views/admins/_admin_bar.haml @@ -6,6 +6,5 @@ %li= link_to t('.weekly_user_stats'), weekly_user_stats_path %li= link_to t('.pod_stats'), pod_stats_path %li= link_to t('.correlations'), correlations_path - - if AppConfig.admins.inline_resque_web? - %li= link_to t('.resque_overview'), resque_web_path + %li= link_to t('.sidekiq_monitor'), sidekiq_path diff --git a/app/workers/base.rb b/app/workers/base.rb new file mode 100644 index 000000000..f1ee498a6 --- /dev/null +++ b/app/workers/base.rb @@ -0,0 +1,23 @@ +# Copyright (c) 2010-2011, Diaspora Inc. This file is +# licensed under the Affero General Public License version 3 or later. See +# the COPYRIGHT file. + +module Workers + class Base + include Sidekiq::Worker + sidekiq_options timeout: AppConfig.environment.sidekiq.timeout.to_i, + backtrace: ((bt = AppConfig.environment.sidekiq.backtrace.get) && bt.to_i), + retry: AppConfig.environment.sidekiq.retry.to_i + + # In the long term we need to eliminate the cause of these + def suppress_annoying_errors(&block) + yield + rescue Diaspora::ContactRequiredUnlessRequest, + Diaspora::RelayableObjectWithoutParent => e + Rails.logger.info("error on receive: #{e.class}") + rescue ActiveRecord::RecordInvalid => e + Rails.logger.info("failed to save received object: #{e.record.errors.full_messages}") + raise e unless e.message.match(/already been taken/) + end + end +end diff --git a/app/models/jobs/deferred_dispatch.rb b/app/workers/deferred_dispatch.rb similarity index 85% rename from app/models/jobs/deferred_dispatch.rb rename to app/workers/deferred_dispatch.rb index e0b43fcbf..1833ce9b9 100644 --- a/app/models/jobs/deferred_dispatch.rb +++ b/app/workers/deferred_dispatch.rb @@ -2,11 +2,11 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -module Jobs +module Workers class DeferredDispatch < Base - @queue = :dispatch + sidekiq_options queue: :dispatch - def self.perform(user_id, object_class_name, object_id, opts) + def perform(user_id, object_class_name, object_id, opts) user = User.find(user_id) object = object_class_name.constantize.find(object_id) opts = HashWithIndifferentAccess.new(opts) diff --git a/app/models/jobs/delete_account.rb b/app/workers/delete_account.rb similarity index 75% rename from app/models/jobs/delete_account.rb rename to app/workers/delete_account.rb index 0d856dda4..ef3991f29 100644 --- a/app/models/jobs/delete_account.rb +++ b/app/workers/delete_account.rb @@ -3,10 +3,11 @@ # the COPYRIGHT file. -module Jobs +module Workers class DeleteAccount < Base - @queue = :delete_account - def self.perform(account_deletion_id) + sidekiq_options queue: :delete_account + + def perform(account_deletion_id) account_deletion = AccountDeletion.find(account_deletion_id) account_deletion.perform! end diff --git a/app/models/jobs/delete_post_from_service.rb b/app/workers/delete_post_from_service.rb similarity index 75% rename from app/models/jobs/delete_post_from_service.rb rename to app/workers/delete_post_from_service.rb index b5a8302a7..b790f8d88 100644 --- a/app/models/jobs/delete_post_from_service.rb +++ b/app/workers/delete_post_from_service.rb @@ -2,11 +2,11 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. # -module Jobs +module Workers class DeletePostFromService < Base - @queue = :http_service + sidekiq_options queue: :http_service - def self.perform(service_id, service_post_id) + def perform(service_id, service_post_id) service = Service.find_by_id(service_id) service.delete_post(service_post_id) end diff --git a/app/models/jobs/fetch_profile_photo.rb b/app/workers/fetch_profile_photo.rb similarity index 87% rename from app/models/jobs/fetch_profile_photo.rb rename to app/workers/fetch_profile_photo.rb index d750ee3ba..bea7e104c 100644 --- a/app/models/jobs/fetch_profile_photo.rb +++ b/app/workers/fetch_profile_photo.rb @@ -3,10 +3,11 @@ # the COPYRIGHT file. -module Jobs +module Workers class FetchProfilePhoto < Base - @queue = :photos - def self.perform(user_id, service_id, fallback_image_url = nil) + sidekiq_options queue: :photos + + def perform(user_id, service_id, fallback_image_url = nil) service = Service.find(service_id) image_url = service.profile_photo_url diff --git a/app/models/jobs/fetch_public_posts.rb b/app/workers/fetch_public_posts.rb similarity index 53% rename from app/models/jobs/fetch_public_posts.rb rename to app/workers/fetch_public_posts.rb index df7f0e959..8b5ca00c8 100644 --- a/app/models/jobs/fetch_public_posts.rb +++ b/app/workers/fetch_public_posts.rb @@ -2,14 +2,12 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -module Jobs +module Workers class FetchPublicPosts < Base - @queue = :http_service + sidekiq_options queue: :http_service - def self.perform(diaspora_id) - require Rails.root.join('lib','diaspora','fetcher','public') - - PublicFetcher.new.fetch!(diaspora_id) + def perform(diaspora_id) + Diaspora::Fetcher::Public.new.fetch!(diaspora_id) end end end diff --git a/app/models/jobs/fetch_webfinger.rb b/app/workers/fetch_webfinger.rb similarity index 64% rename from app/models/jobs/fetch_webfinger.rb rename to app/workers/fetch_webfinger.rb index 9c96c41ee..dd88db32c 100644 --- a/app/models/jobs/fetch_webfinger.rb +++ b/app/workers/fetch_webfinger.rb @@ -2,15 +2,15 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -module Jobs +module Workers class FetchWebfinger < Base - @queue = :socket_webfinger + sidekiq_options queue: :socket_webfinger - def self.perform(account) + def perform(account) person = Webfinger.new(account).fetch # also, schedule to fetch a few public posts from that person - Resque.enqueue(Jobs::FetchPublicPosts, person.diaspora_handle) unless person.nil? + Workers::FetchPublicPosts.perform_async(person.diaspora_handle) unless person.nil? end end end diff --git a/app/workers/gather_o_embed_data.rb b/app/workers/gather_o_embed_data.rb new file mode 100644 index 000000000..2f8c90119 --- /dev/null +++ b/app/workers/gather_o_embed_data.rb @@ -0,0 +1,22 @@ +# Copyright (c) 2010-2011, Diaspora Inc. This file is +# licensed under the Affero General Public License version 3 or later. See +# the COPYRIGHT file. +# + +module Workers + class GatherOEmbedData < Base + sidekiq_options queue: :http_service + + def perform(post_id, url, retry_count=1) + post = Post.find(post_id) + post.o_embed_cache = OEmbedCache.find_or_create_by_url(url) + post.save + rescue ActiveRecord::RecordNotFound + # User created a post and deleted it right afterwards before we + # we had a chance to run the job. + # On the other hand sometimes the job runs before the Post is + # fully persisted. So we just reduce the amount of retries. + GatherOEmbedData.perform_in(1.minute, post_id, url, retry_count+1) unless retry_count > 3 + end + end +end diff --git a/app/models/jobs/http_multi.rb b/app/workers/http_multi.rb similarity index 60% rename from app/models/jobs/http_multi.rb rename to app/workers/http_multi.rb index 1132fcbe4..2d633a1fe 100644 --- a/app/models/jobs/http_multi.rb +++ b/app/workers/http_multi.rb @@ -2,29 +2,25 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -require 'uri' -require Rails.root.join('lib', 'hydra_wrapper') - -module Jobs +module Workers class HttpMulti < Base - - @queue = :http + sidekiq_options queue: :http MAX_RETRIES = 3 - def self.perform(user_id, encoded_object_xml, person_ids, dispatcher_class_as_string, retry_count=0) + def perform(user_id, encoded_object_xml, person_ids, dispatcher_class_as_string, retry_count=0) user = User.find(user_id) people = Person.where(:id => person_ids) dispatcher = dispatcher_class_as_string.constantize - hydra = ::HydraWrapper.new(user, people, encoded_object_xml, dispatcher) + hydra = HydraWrapper.new(user, people, encoded_object_xml, dispatcher) hydra.enqueue_batch hydra.run unless hydra.failed_people.empty? if retry_count < MAX_RETRIES - Resque.enqueue(Jobs::HttpMulti, user_id, encoded_object_xml, hydra.failed_people, dispatcher_class_as_string, retry_count + 1 ) + Workers::HttpMulti.perform_in(1.hour, user_id, encoded_object_xml, hydra.failed_people, dispatcher_class_as_string, retry_count + 1) else Rails.logger.info("event=http_multi_abandon sender_id=#{user_id} failed_recipient_ids='[#{person_ids.join(', ')}] '") end diff --git a/app/models/jobs/mail/also_commented.rb b/app/workers/mail/also_commented.rb similarity index 64% rename from app/models/jobs/mail/also_commented.rb rename to app/workers/mail/also_commented.rb index 6e53ff6cd..617c1961a 100644 --- a/app/models/jobs/mail/also_commented.rb +++ b/app/workers/mail/also_commented.rb @@ -1,8 +1,9 @@ -module Jobs +module Workers module Mail class AlsoCommented < Base - @queue = :mail - def self.perform(recipient_id, sender_id, comment_id) + sidekiq_options queue: :mail + + def perform(recipient_id, sender_id, comment_id) if email = Notifier.also_commented(recipient_id, sender_id, comment_id) email.deliver end diff --git a/app/models/jobs/mail/comment_on_post.rb b/app/workers/mail/comment_on_post.rb similarity index 58% rename from app/models/jobs/mail/comment_on_post.rb rename to app/workers/mail/comment_on_post.rb index ddc613571..5ec07782a 100644 --- a/app/models/jobs/mail/comment_on_post.rb +++ b/app/workers/mail/comment_on_post.rb @@ -1,8 +1,9 @@ -module Jobs +module Workers module Mail class CommentOnPost < Base - @queue = :mail - def self.perform(recipient_id, sender_id, comment_id) + sidekiq_options queue: :mail + + def perform(recipient_id, sender_id, comment_id) Notifier.comment_on_post(recipient_id, sender_id, comment_id).deliver end end diff --git a/app/models/jobs/mail/confirm_email.rb b/app/workers/mail/confirm_email.rb similarity index 58% rename from app/models/jobs/mail/confirm_email.rb rename to app/workers/mail/confirm_email.rb index 6eeb1b1e1..ec7c6742e 100644 --- a/app/models/jobs/mail/confirm_email.rb +++ b/app/workers/mail/confirm_email.rb @@ -1,8 +1,9 @@ -module Jobs +module Workers module Mail class ConfirmEmail < Base - @queue = :mail - def self.perform(user_id) + sidekiq_options queue: :mail + + def perform(user_id) Notifier.confirm_email(user_id).deliver end end diff --git a/app/models/jobs/mail/invite_user_by_email.rb b/app/workers/mail/invite_user_by_email.rb similarity index 80% rename from app/models/jobs/mail/invite_user_by_email.rb rename to app/workers/mail/invite_user_by_email.rb index e1e348a3f..87946cde5 100644 --- a/app/models/jobs/mail/invite_user_by_email.rb +++ b/app/workers/mail/invite_user_by_email.rb @@ -2,12 +2,12 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. - -module Jobs +module Workers module Mail class InviteUserByEmail < Base - @queue = :mail - def self.perform(invite_id) + sidekiq_options queue: :mail + + def perform(invite_id) invite = Invitation.find(invite_id) I18n.with_locale(invite.language) do invite.send! diff --git a/app/models/jobs/mail/liked.rb b/app/workers/mail/liked.rb similarity index 54% rename from app/models/jobs/mail/liked.rb rename to app/workers/mail/liked.rb index 24b434aa6..195c73622 100644 --- a/app/models/jobs/mail/liked.rb +++ b/app/workers/mail/liked.rb @@ -1,8 +1,9 @@ -module Jobs +module Workers module Mail class Liked < Base - @queue = :mail - def self.perform(recipient_id, sender_id, like_id) + sidekiq_options queue: :mail + + def perform(recipient_id, sender_id, like_id) Notifier.liked(recipient_id, sender_id, like_id).deliver end end diff --git a/app/models/jobs/mail/mentioned.rb b/app/workers/mail/mentioned.rb similarity index 73% rename from app/models/jobs/mail/mentioned.rb rename to app/workers/mail/mentioned.rb index 4a91f01dc..950063ec6 100644 --- a/app/models/jobs/mail/mentioned.rb +++ b/app/workers/mail/mentioned.rb @@ -3,11 +3,12 @@ # the COPYRIGHT file. -module Jobs +module Workers module Mail class Mentioned < Base - @queue = :mail - def self.perform(recipient_id, actor_id, target_id) + sidekiq_options queue: :mail + + def perform(recipient_id, actor_id, target_id) Notifier.mentioned( recipient_id, actor_id, target_id).deliver end end diff --git a/app/models/jobs/mail/private_message.rb b/app/workers/mail/private_message.rb similarity index 73% rename from app/models/jobs/mail/private_message.rb rename to app/workers/mail/private_message.rb index ae4d71a1d..3fe953539 100644 --- a/app/models/jobs/mail/private_message.rb +++ b/app/workers/mail/private_message.rb @@ -3,11 +3,12 @@ # the COPYRIGHT file. -module Jobs +module Workers module Mail class PrivateMessage < Base - @queue = :mail - def self.perform(recipient_id, actor_id, target_id) + sidekiq_options queue: :mail + + def perform(recipient_id, actor_id, target_id) Notifier.private_message( recipient_id, actor_id, target_id).deliver end end diff --git a/app/models/jobs/mail/reshared.rb b/app/workers/mail/reshared.rb similarity index 55% rename from app/models/jobs/mail/reshared.rb rename to app/workers/mail/reshared.rb index 8c768236e..97d74bb61 100644 --- a/app/models/jobs/mail/reshared.rb +++ b/app/workers/mail/reshared.rb @@ -1,8 +1,9 @@ -module Jobs +module Workers module Mail class Reshared < Base - @queue = :mail - def self.perform(recipient_id, sender_id, reshare_id) + sidekiq_options queue: :mail + + def perform(recipient_id, sender_id, reshare_id) Notifier.reshared(recipient_id, sender_id, reshare_id).deliver end end diff --git a/app/models/jobs/mail/started_sharing.rb b/app/workers/mail/started_sharing.rb similarity index 73% rename from app/models/jobs/mail/started_sharing.rb rename to app/workers/mail/started_sharing.rb index 4fa482146..6ca50f96a 100644 --- a/app/models/jobs/mail/started_sharing.rb +++ b/app/workers/mail/started_sharing.rb @@ -3,11 +3,12 @@ # the COPYRIGHT file. -module Jobs +module Workers module Mail class StartedSharing < Base - @queue = :mail - def self.perform(recipient_id, sender_id, target_id) + sidekiq_options queue: :mail + + def perform(recipient_id, sender_id, target_id) Notifier.started_sharing(recipient_id, sender_id).deliver end end diff --git a/app/models/jobs/notify_local_users.rb b/app/workers/notify_local_users.rb similarity index 79% rename from app/models/jobs/notify_local_users.rb rename to app/workers/notify_local_users.rb index aa946f7f1..08466aba2 100644 --- a/app/models/jobs/notify_local_users.rb +++ b/app/workers/notify_local_users.rb @@ -2,13 +2,11 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -module Jobs +module Workers class NotifyLocalUsers < Base - @queue = :receive_local + sidekiq_options queue: :receive_local - require Rails.root.join('app', 'models', 'notification') - - def self.perform(user_ids, object_klass, object_id, person_id) + def perform(user_ids, object_klass, object_id, person_id) object = object_klass.constantize.find_by_id(object_id) diff --git a/app/models/jobs/post_to_service.rb b/app/workers/post_to_service.rb similarity index 76% rename from app/models/jobs/post_to_service.rb rename to app/workers/post_to_service.rb index c4f356701..24cf053db 100644 --- a/app/models/jobs/post_to_service.rb +++ b/app/workers/post_to_service.rb @@ -2,11 +2,11 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. # -module Jobs +module Workers class PostToService < Base - @queue = :http_service + sidekiq_options queue: :http_service - def self.perform(service_id, post_id, url) + def perform(service_id, post_id, url) service = Service.find_by_id(service_id) post = Post.find_by_id(post_id) service.post(post, url) diff --git a/app/models/jobs/process_photo.rb b/app/workers/process_photo.rb similarity index 86% rename from app/models/jobs/process_photo.rb rename to app/workers/process_photo.rb index 048e13c88..00d959962 100644 --- a/app/models/jobs/process_photo.rb +++ b/app/workers/process_photo.rb @@ -3,10 +3,11 @@ # the COPYRIGHT file. -module Jobs +module Workers class ProcessPhoto < Base - @queue = :photos - def self.perform(id) + sidekiq_options queue: :photos + + def perform(id) photo = Photo.find(id) unprocessed_image = photo.unprocessed_image diff --git a/app/models/jobs/publish_to_hub.rb b/app/workers/publish_to_hub.rb similarity index 71% rename from app/models/jobs/publish_to_hub.rb rename to app/workers/publish_to_hub.rb index 80858984a..055bd0213 100644 --- a/app/models/jobs/publish_to_hub.rb +++ b/app/workers/publish_to_hub.rb @@ -2,12 +2,11 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -module Jobs +module Workers class PublishToHub < Base - @queue = :http_service + sidekiq_options queue: :http_service - def self.perform(sender_public_url) - require Rails.root.join('lib', 'pubsubhubbub') + def perform(sender_public_url) atom_url = sender_public_url + '.atom' Pubsubhubbub.new(AppConfig.environment.pubsub_server.get).publish(atom_url) end diff --git a/app/workers/receive.rb b/app/workers/receive.rb new file mode 100644 index 000000000..3eba79442 --- /dev/null +++ b/app/workers/receive.rb @@ -0,0 +1,19 @@ +# Copyright (c) 2010-2011, Diaspora Inc. This file is +# licensed under the Affero General Public License version 3 or later. See +# the COPYRIGHT file. + + +module Workers + class Receive < Base + sidekiq_options queue: :receive + + def perform(user_id, xml, salmon_author_id) + suppress_annoying_errors do + user = User.find(user_id) + salmon_author = Person.find(salmon_author_id) + zord = Postzord::Receiver::Private.new(user, :person => salmon_author) + zord.parse_and_receive(xml) + end + end + end +end diff --git a/app/models/jobs/receive_encrypted_salmon.rb b/app/workers/receive_encrypted_salmon.rb similarity index 73% rename from app/models/jobs/receive_encrypted_salmon.rb rename to app/workers/receive_encrypted_salmon.rb index 6f14e8f1a..ffb3f6df4 100644 --- a/app/models/jobs/receive_encrypted_salmon.rb +++ b/app/workers/receive_encrypted_salmon.rb @@ -3,12 +3,11 @@ # the COPYRIGHT file. -require Rails.root.join('lib', 'postzord', 'receiver', 'private') -module Jobs +module Workers class ReceiveEncryptedSalmon < Base - @queue = :receive_salmon + sidekiq_options queue: :receive_salmon - def self.perform(user_id, xml) + def perform(user_id, xml) suppress_annoying_errors do user = User.find(user_id) zord = Postzord::Receiver::Private.new(user, :salmon_xml => xml) diff --git a/app/models/jobs/receive_local_batch.rb b/app/workers/receive_local_batch.rb similarity index 60% rename from app/models/jobs/receive_local_batch.rb rename to app/workers/receive_local_batch.rb index 99968fbe4..81d8a9ecf 100644 --- a/app/models/jobs/receive_local_batch.rb +++ b/app/workers/receive_local_batch.rb @@ -2,15 +2,11 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -require Rails.root.join('lib', 'postzord', 'receiver', 'private') -require Rails.root.join('lib', 'postzord', 'receiver', 'local_batch') - -module Jobs +module Workers class ReceiveLocalBatch < Base + sidekiq_options queue: :receive - @queue = :receive - - def self.perform(object_class_string, object_id, recipient_user_ids) + def perform(object_class_string, object_id, recipient_user_ids) object = object_class_string.constantize.find(object_id) receiver = Postzord::Receiver::LocalBatch.new(object, recipient_user_ids) receiver.perform! diff --git a/app/workers/receive_unencrypted_salmon.rb b/app/workers/receive_unencrypted_salmon.rb new file mode 100644 index 000000000..2f7085730 --- /dev/null +++ b/app/workers/receive_unencrypted_salmon.rb @@ -0,0 +1,21 @@ +# Copyright (c) 2010-2011, Diaspora Inc. This file is +# licensed under the Affero General Public License version 3 or later. See +# the COPYRIGHT file. + +module Workers + class ReceiveUnencryptedSalmon < Base + sidekiq_options queue: :receive + + def perform(xml) + suppress_annoying_errors do + begin + receiver = Postzord::Receiver::Public.new(xml) + receiver.perform! + rescue => e + FEDERATION_LOGGER.info(e.message) + raise e + end + end + end + end +end diff --git a/app/models/jobs/resend_invitation.rb b/app/workers/resend_invitation.rb similarity index 76% rename from app/models/jobs/resend_invitation.rb rename to app/workers/resend_invitation.rb index 0b06ea63e..b2a0cc00e 100644 --- a/app/models/jobs/resend_invitation.rb +++ b/app/workers/resend_invitation.rb @@ -3,10 +3,11 @@ # the COPYRIGHT file. -module Jobs +module Workers class ResendInvitation < Base - @queue = :mail - def self.perform(invitation_id) + sidekiq_options queue: :mail + + def perform(invitation_id) inv = Invitation.find(invitation_id) inv.resend end diff --git a/app/models/jobs/reset_password.rb b/app/workers/reset_password.rb similarity index 66% rename from app/models/jobs/reset_password.rb rename to app/workers/reset_password.rb index 7b3e50e96..a0b869f59 100644 --- a/app/models/jobs/reset_password.rb +++ b/app/workers/reset_password.rb @@ -1,8 +1,8 @@ -module Jobs +module Workers class ResetPassword < Base - @queue = :mail + sidekiq_options queue: :mail - def self.perform(user_id) + def perform(user_id) user = User.find(user_id) ::Devise.mailer.reset_password_instructions(user).deliver end diff --git a/config/database.yml.example b/config/database.yml.example index ddda09fea..baec2fbbf 100644 --- a/config/database.yml.example +++ b/config/database.yml.example @@ -18,10 +18,13 @@ postgres: &postgres # Comment the the mysql line and uncomment the postgres line # if you want to use postgres -choose: &choose +common: &common # Choose one of the following <<: *mysql #<<: *postgres + + # Should match environment.sidekiq.concurrency + #pool: 25 ################################################## #### CONFIGURE ABOVE ############################# @@ -32,20 +35,20 @@ choose: &choose postgres_travis: &postgres_travis adapter: postgresql username: postgres -common: &common - <<: *choose -development: +combined: &combined <<: *common +development: + <<: *combined database: diaspora_development production: - <<: *common + <<: *combined database: diaspora_production test: - <<: *common + <<: *combined database: "diaspora_test" integration1: - <<: *common + <<: *combined database: diaspora_integration1 integration2: - <<: *common + <<: *combined database: diaspora_integration2 diff --git a/config/defaults.yml b/config/defaults.yml index 19525e6fd..e1dad5b27 100644 --- a/config/defaults.yml +++ b/config/defaults.yml @@ -12,6 +12,13 @@ defaults: redis: require_ssl: true single_process_mode: false + sidekiq: + namespace: "diaspora" + concurrency: 5 + retry: 10 + timeout: 900 + backtrace: 15 + log: "log/sidekiq.log" s3: enable: false key: @@ -32,8 +39,7 @@ defaults: stdout_log: database: 'mysql' unicorn_worker: 2 - embed_resque_worker: false - resque_workers: 1 + embed_sidekiq_worker: false privacy: jquery_cdn: true google_analytics_key: @@ -88,7 +94,6 @@ defaults: admins: account: podmin_email: - inline_resque_web: true development: environment: @@ -123,12 +128,14 @@ test: integration1: environment: url: "http://localhost:45789/" + single_process_mode: true assets: serve: true require_ssl: false integration2: environment: url: "http://localhost:34658/" + single_process_mode: true assets: serve: true require_ssl: false diff --git a/config/diaspora.yml.example b/config/diaspora.yml.example index 2bee8212d..355dbf2aa 100644 --- a/config/diaspora.yml.example +++ b/config/diaspora.yml.example @@ -66,6 +66,38 @@ configuration: ## Section ## all the time intensive jobs must be run inside the request cycle. ## So this is higly unrecommended for production setups. #single_process_mode: true + + ## Sidekiq - background processing + sidekiq: ## Section + ## Number of parallel threads Sidekiq uses + ## If you touch this please set the pool setting + ## in your database.yml to a value that's at minimum + ## close to this! The default value is 5 but you can safely + ## increase it to 25 and more on a medium sized pod. + ## This applies per started Sidekiq worker, so if you set it to + ## 25 and start two workers you'll process up to 50 jobs in parallel. + #concurrency: 25 + + ## Number of times a job is retried + ## There's an exponential backoff, if you set this too + ## high you might get too many jobs in the queue + ## Set this to false to disable it completely + #retry: 10 + + ## Time in seconds before a job is killed + #timeout: 900 + + ## Namespace to use in Redis, useful if you need to run + ## multiple instances of Diaspora using the same Redis instance + #namespace: "diaspora" + + ## Lines of backtrace that is stored on failure + ## Set this to false if you're not interested in this data to + ## reduce memory usage (and log size) + #backtrace: 15 + + ## Log file for Sidekiq + #log: "log/sidekiq.log" ## Use Amazon S3 instead of your local filesystem ## to handle uploaded pictures. @@ -124,12 +156,9 @@ configuration: ## Section ## you have many users #unicorn_worker: 2 - ## Embed a resque worker inside the unicorn process, useful for + ## Embed a Sidekiq worker inside the unicorn process, useful for ## minimal Heroku setups - #embed_resque_worker: true - - ## Number of resque workers to start - #resque_workers: 1 + #embed_sidekiq_worker: true ## Settings probably affecting the privacy of your users privacy: ## Section @@ -293,11 +322,6 @@ configuration: ## Section ## E-Mail address users can contact the administrator #podmin_email: 'podmin@example.org' - - ## Resque is the background processing system used by Diaspora - ## Resque web is an admin tool for it. This settings decides whether - ## or not to inline it into Diaspora. - #inline_resque_web: true ## Here you can make overides to settings defined above if you need ## to have them different in different environments. diff --git a/config/initializers/load_mail_jobs.rb b/config/initializers/load_mail_jobs.rb deleted file mode 100644 index be356724f..000000000 --- a/config/initializers/load_mail_jobs.rb +++ /dev/null @@ -1 +0,0 @@ -Dir[Rails.root.join('app', 'models', 'jobs', 'mail', '*.rb')].each { |file| require file } \ No newline at end of file diff --git a/config/initializers/resque.rb b/config/initializers/resque.rb deleted file mode 100644 index 1ed9eaf0f..000000000 --- a/config/initializers/resque.rb +++ /dev/null @@ -1,36 +0,0 @@ -require 'resque' - -Resque::Plugins::Timeout.timeout = 300 - -if !AppConfig.environment.single_process_mode? - Resque.redis = AppConfig.get_redis_instance -end - -# Single process-mode hooks using Resque.inline -if AppConfig.environment.single_process_mode? - if Rails.env == 'production' - puts "WARNING: You are running Diaspora in production without Resque" - puts " workers turned on. Please set single_process_mode to false in" - puts " config/diaspora.yml." - end - Resque.inline = true -end - -if AppConfig.admins.monitoring.airbrake_api_key.present? - require 'resque/failure/multiple' - require 'resque/failure/airbrake' - require 'resque/failure/redis' - Resque::Failure::Airbrake.configure do |config| - config.api_key = AppConfig.admins.monitoring.airbrake_api_key - config.secure = true - end - Resque::Failure::Multiple.classes = [Resque::Failure::Redis, Resque::Failure::Airbrake] - Resque::Failure.backend = Resque::Failure::Multiple -end - - -if AppConfig.admins.inline_resque_web? - require 'resque/server' - require Rails.root.join('lib', 'admin_rack') - Resque::Server.use AdminRack -end diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb new file mode 100644 index 000000000..01f82dbd8 --- /dev/null +++ b/config/initializers/sidekiq.rb @@ -0,0 +1,50 @@ +require 'sidekiq_middlewares' + +# Single process-mode +if AppConfig.environment.single_process_mode? && Rails.env != "test" + if Rails.env == 'production' + puts "WARNING: You are running Diaspora in production without Sidekiq" + puts " workers turned on. Please set single_process_mode to false in" + puts " config/diaspora.yml." + end + require 'sidekiq/testing/inline' +end + + +Sidekiq.configure_server do |config| + config.redis = AppConfig.get_redis_options + + config.options = config.options.merge({ + concurrency: AppConfig.environment.sidekiq.concurrency.to_i, + queues: %w{ + socket_webfinger + photos + http_service + dispatch + mail + delete_account + receive_local + receive + receive_salmon + http + default + } + }) + + config.server_middleware do |chain| + chain.add SidekiqMiddlewares::CleanAndShortBacktraces + end + + Sidekiq::Logging.initialize_logger AppConfig.sidekiq_log unless AppConfig.heroku? + + # Set connection pool on Heroku + database_url = ENV['DATABASE_URL'] + if(database_url) + ENV['DATABASE_URL'] = "#{database_url}?pool=#{AppConfig.environment.sidekiq.concurrency.get}" + ActiveRecord::Base.establish_connection + end +end + +Sidekiq.configure_client do |config| + config.redis = AppConfig.get_redis_options +end diff --git a/config/locales/diaspora/en.yml b/config/locales/diaspora/en.yml index f5c86971c..d191cffd5 100644 --- a/config/locales/diaspora/en.yml +++ b/config/locales/diaspora/en.yml @@ -89,7 +89,7 @@ en: weekly_user_stats: "Weekly User Stats" pod_stats: "Pod Stats" correlations: "Correlations" - resque_overview: "Resque Overview" + sidekiq_monitor: "Sidekiq monitor" correlations: correlations_count: "Correlations with Sign In Count:" user_search: diff --git a/config/routes.rb b/config/routes.rb index 57e41286c..35174d99d 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -2,11 +2,17 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. +require 'sidekiq/web' + Diaspora::Application.routes.draw do if Rails.env.production? mount RailsAdmin::Engine => '/admin_panel', :as => 'rails_admin' end + constraints ->(req) { req.env["warden"].authenticate?(scope: :user) && + req.env['warden'].user.admin? } do + mount Sidekiq::Web => '/sidekiq', :as => 'sidekiq' + end get "/atom.xml" => redirect('http://blog.diasporafoundation.org/feed/atom') #too many stupid redirects :() @@ -206,11 +212,6 @@ Diaspora::Application.routes.draw do #Protocol Url get 'protocol' => redirect("https://github.com/diaspora/diaspora/wiki/Diaspora%27s-federation-protocol") - # Resque web - if AppConfig.admins.inline_resque_web? - mount Resque::Server.new, :at => '/resque-jobs', :as => "resque_web" - end - # Startpage root :to => 'home#show' end diff --git a/config/unicorn.rb b/config/unicorn.rb index b63d252e9..281469169 100644 --- a/config/unicorn.rb +++ b/config/unicorn.rb @@ -15,7 +15,7 @@ preload_app true # How long to wait before killing an unresponsive worker timeout 30 -@resque_pid = nil +@sidekiq_pid = nil #pid '/var/run/diaspora/diaspora.pid' #listen '/var/run/diaspora/diaspora.sock', :backlog => 2048 @@ -33,14 +33,12 @@ before_fork do |server, worker| ActiveRecord::Base.connection.disconnect! # disconnect redis if in use - if !AppConfig.single_process_mode? - Resque.redis.client.disconnect + unless AppConfig.single_process_mode? + Sidekiq.redis {|redis| redis.client.disconnect } end - if AppConfig.server.embed_resque_worker? - # Clean up Resque workers killed by previous deploys/restarts - Resque.workers.each { |w| w.unregister_worker } - @resque_pid ||= spawn('bundle exec rake resque:work QUEUES=*') + if AppConfig.server.embed_sidekiq_worker? + @sidekiq_pid ||= spawn('bundle exec sidekiq') end old_pid = '/var/run/diaspora/diaspora.pid.oldbin' @@ -58,9 +56,8 @@ after_fork do |server, worker| # If using preload_app, enable this line ActiveRecord::Base.establish_connection - # copy pasta from resque.rb because i'm a bad person - if !AppConfig.environment.single_process_mode? - Resque.redis = AppConfig.get_redis_instance + unless AppConfig.environment.single_process_mode? + Sidekiq.redis = AppConfig.get_redis_options end # Enable this line to have the workers run as different user/group diff --git a/db/seeds.rb b/db/seeds.rb index 0aaedf48c..7cd2f1949 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -46,7 +46,7 @@ Role.add_admin(bob.person) puts "done!" -require Rails.root.join('spec', 'support', 'fake_resque') +require 'sidekiq/testing/inline' require Rails.root.join('spec', 'support', 'user_methods') print "Seeding post data..." diff --git a/features/support/env.rb b/features/support/env.rb index ac4b44c54..c95bb5eab 100644 --- a/features/support/env.rb +++ b/features/support/env.rb @@ -46,16 +46,16 @@ prefork = proc do require File.join(File.dirname(__FILE__), "integration_sessions_controller") require File.join(File.dirname(__FILE__), "poor_mans_webmock") + require 'sidekiq/testing/inline' + require Rails.root.join('spec', 'helper_methods') + require Rails.root.join('spec', 'support', 'inlined_jobs') require Rails.root.join('spec', 'support', 'user_methods') include HelperMethods # require 'webmock/cucumber' # WebMock.disable_net_connect!(:allow_localhost => true) - require Rails.root.join('spec', 'support', 'fake_resque') - - require File.join(File.dirname(__FILE__), 'run_resque_in_process') #hax to get rubymine to run spork, set RUBYMINE_HOME in your .bash_profile if ENV["RUBYMINE_HOME"] diff --git a/features/support/poor_mans_webmock.rb b/features/support/poor_mans_webmock.rb index 487696ee3..5b04d5c05 100644 --- a/features/support/poor_mans_webmock.rb +++ b/features/support/poor_mans_webmock.rb @@ -2,32 +2,28 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. -module Jobs +module Workers class PublishToHub < Base - @queue = :http_service - def self.perform(sender_public_url) + def perform(sender_public_url) # don't publish to pubsubhubbub in cucumber end end class HttpMulti < Base - @queue = :http - def self.perform(user_id, enc_object_xml, person_ids, retry_count=0) + def perform(user_id, enc_object_xml, person_ids, retry_count=0) # don't federate in cucumber end end class HttpPost < Base - @queue = :http - def self.perform(url, body, tries_remaining = NUM_TRIES) + def perform(url, body, tries_remaining = NUM_TRIES) # don't post to outside services in cucumber end end class PostToService < Base - @queue = :http_service - def self.perform(service_id, post_id, url) + def perform(service_id, post_id, url) # don't post to services in cucumber end end -end \ No newline at end of file +end diff --git a/features/support/run_resque_in_process.rb b/features/support/run_resque_in_process.rb deleted file mode 100644 index c2b10aa45..000000000 --- a/features/support/run_resque_in_process.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Resque - def enqueue(klass, *args) - klass.send(:perform, *args) - end -end \ No newline at end of file diff --git a/lib/admin_rack.rb b/lib/admin_rack.rb deleted file mode 100644 index 39253b3d7..000000000 --- a/lib/admin_rack.rb +++ /dev/null @@ -1,19 +0,0 @@ -class AdminRack - def initialize(app) - @app = app - end - - def call(env) - user = env['warden'].authenticate(:scope => :user) - if user && user.admin? - @app.call(env) - else - [307, {"Location" => '/'}, self] - end - end - - def each(&block) - end -end - - diff --git a/lib/configuration_methods.rb b/lib/configuration_methods.rb index 095ca854f..1dc2b2b19 100644 --- a/lib/configuration_methods.rb +++ b/lib/configuration_methods.rb @@ -74,7 +74,7 @@ module Configuration (git_revision || version)[0..8] end - def get_redis_instance + def get_redis_options if redistogo_url.present? $stderr.puts "WARNING: using the REDISTOGO_URL environment variable is deprecated, please use REDIS_URL now." ENV['REDIS_URL'] = redistogo_url @@ -85,17 +85,25 @@ module Configuration redis_url = ENV['REDIS_URL'] || environment.redis.get if ENV['RAILS_ENV']== 'integration2' - redis_options = { :host => 'localhost', :port => 6380 } + redis_options[:url] = "redis://localhost:6380" elsif redis_url.present? unless redis_url.start_with?("redis://") || redis_url.start_with?("unix:///") $stderr.puts "WARNING: Your redis url (#{redis_url}) doesn't start with redis:// or unix:///" end - redis_options = { :url => redis_url } + redis_options[:url] = redis_url end - Redis.new(redis_options.merge(:thread_safe => true)) + redis_options[:namespace] = AppConfig.environment.sidekiq_namespace.get + + redis_options end + def sidekiq_log + path = Pathname.new environment.sidekiq.log.get + path = Rails.root.join(path) unless pathname.absolute? + path.to_s + end + private def get_git_info diff --git a/lib/diaspora/exceptions.rb b/lib/diaspora/exceptions.rb index 79b5283bb..eba8366e0 100644 --- a/lib/diaspora/exceptions.rb +++ b/lib/diaspora/exceptions.rb @@ -17,4 +17,17 @@ module Diaspora class NotMine < StandardError end + + # Received a message without having a contact + class ContactRequiredUnlessRequest < StandardError + end + + # Got a relayable (comment, like etc.) without having the parent + class RelayableObjectWithoutParent < StandardError + end + + # After building an object the author doesn't match the one in the + # original XML message + class AuthorXMLAuthorMismatch < StandardError + end end diff --git a/lib/diaspora/fetcher/public.rb b/lib/diaspora/fetcher/public.rb index 409d3ffe8..8f196c625 100644 --- a/lib/diaspora/fetcher/public.rb +++ b/lib/diaspora/fetcher/public.rb @@ -75,7 +75,7 @@ module Diaspora; module Fetcher; class Public FEDERATION_LOGGER.info "fetching public posts for #{@person.diaspora_handle}" - resp = Faraday.get("#{@person.url}/people/#{@person.guid}") do |req| + resp = Faraday.get("#{@person.url}people/#{@person.guid}") do |req| req.headers[:accept] = 'application/json' req.headers[:user_agent] = 'diaspora-fetcher' end diff --git a/lib/diaspora/parser.rb b/lib/diaspora/parser.rb index 757bd13dd..1d2c50092 100644 --- a/lib/diaspora/parser.rb +++ b/lib/diaspora/parser.rb @@ -13,7 +13,7 @@ module Diaspora rescue NameError => e # A pods is trying to federate an object we don't recognize. # i.e. their codebase is different from ours. Quietly discard - # so that no Resque job failure is created + # so that no job failure is created nil end end diff --git a/lib/postzord/dispatcher.rb b/lib/postzord/dispatcher.rb index ae6fa2687..a3b986b84 100644 --- a/lib/postzord/dispatcher.rb +++ b/lib/postzord/dispatcher.rb @@ -40,7 +40,7 @@ class Postzord::Dispatcher if opts[:additional_subscribers].present? opts[:additional_subscribers] = [*opts[:additional_subscribers]].map(&:id) end - Resque.enqueue(Jobs::DeferredDispatch, user.id, object.class.to_s, object.id, opts) + Workers::DeferredDispatch.perform_async(user.id, object.class.to_s, object.id, opts) end # @param object [Object] @@ -102,15 +102,16 @@ class Postzord::Dispatcher queue_remote_delivery_job(remote_people) end - # Enqueues a job in Resque + # Enqueues a job # @param remote_people [Array] Recipients of the post on other pods # @return [void] def queue_remote_delivery_job(remote_people) - Resque.enqueue(Jobs::HttpMulti, - @sender.id, - Base64.strict_encode64(@object.to_diaspora_xml), - remote_people.map{|p| p.id}, - self.class.to_s) + Workers::HttpMulti.perform_async( + @sender.id, + Base64.strict_encode64(@object.to_diaspora_xml), + remote_people.map{|p| p.id}, + self.class.to_s + ) end # @param people [Array] Recipients of the post @@ -121,7 +122,7 @@ class Postzord::Dispatcher else people.each do |person| Rails.logger.info("event=push route=local sender=#{@sender.diaspora_handle} recipient=#{person.diaspora_handle} payload_type=#{@object.class}") - Resque.enqueue(Jobs::Receive, person.owner_id, @xml, @sender.person_id) + Workers::Receive.perform_async(person.owner_id, @xml, @sender.person_id) end end end @@ -129,13 +130,13 @@ class Postzord::Dispatcher # @param people [Array] Recipients of the post def batch_deliver_to_local(people) ids = people.map{ |p| p.owner_id } - Resque.enqueue(Jobs::ReceiveLocalBatch, @object.class.to_s, @object.id, ids) + Workers::ReceiveLocalBatch.perform_async(@object.class.to_s, @object.id, ids) Rails.logger.info("event=push route=local sender=#{@sender.diaspora_handle} recipients=#{ids.join(',')} payload_type=#{@object.class}") end def deliver_to_hub Rails.logger.debug("event=post_to_service type=pubsub sender_handle=#{@sender.diaspora_handle}") - Resque.enqueue(Jobs::PublishToHub, @sender.public_url) + Workers::PublishToHub.perform_async(@sender.public_url) end # @param url [String] @@ -146,12 +147,12 @@ class Postzord::Dispatcher end if @object.instance_of?(StatusMessage) services.each do |service| - Resque.enqueue(Jobs::PostToService, service.id, @object.id, url) + Workers::PostToService.perform_async(service.id, @object.id, url) end end if @object.instance_of?(SignedRetraction) services.select { |service| service.respond_to? :delete_post }.each do |service| - Resque.enqueue(Jobs::DeletePostFromService, service.id, @object.target.facebook_id) + Workers::DeletePostFromService.perform_async(service.id, @object.target.facebook_id) end end end @@ -168,7 +169,7 @@ class Postzord::Dispatcher #temp hax unless object_is_related_to_diaspora_hq? - Resque.enqueue(Jobs::NotifyLocalUsers, users.map{|u| u.id}, @object.class.to_s, @object.id, @object.author.id) + Workers::NotifyLocalUsers.perform_async(users.map{|u| u.id}, @object.class.to_s, @object.id, @object.author.id) end end diff --git a/lib/postzord/receiver/private.rb b/lib/postzord/receiver/private.rb index 3e9517073..f2d560e50 100644 --- a/lib/postzord/receiver/private.rb +++ b/lib/postzord/receiver/private.rb @@ -60,12 +60,12 @@ class Postzord::Receiver::Private < Postzord::Receiver end def validate_object - raise "Contact required unless request" if contact_required_unless_request - raise "Relayable object, but no parent object found" if relayable_without_parent? + raise Diaspora::ContactRequiredUnlessRequest if contact_required_unless_request + raise Diaspora::RelayableObjectWithoutParent if relayable_without_parent? assign_sender_handle_if_request - raise "Author does not match XML author" if author_does_not_match_xml_author? + raise Diaspora::AuthorXMLAuthorMismatch if author_does_not_match_xml_author? @object end diff --git a/lib/postzord/receiver/public.rb b/lib/postzord/receiver/public.rb index 43433b28b..ae8cdb869 100644 --- a/lib/postzord/receiver/public.rb +++ b/lib/postzord/receiver/public.rb @@ -36,7 +36,7 @@ class Postzord::Receiver::Public < Postzord::Receiver @object.perform user if user end else - Resque.enqueue(Jobs::ReceiveLocalBatch, @object.class.to_s, @object.id, self.recipient_user_ids) + Workers::ReceiveLocalBatch.perform_async(@object.class.to_s, @object.id, self.recipient_user_ids) true end end @@ -57,7 +57,7 @@ class Postzord::Receiver::Public < Postzord::Receiver def save_object @object = Diaspora::Parser::from_xml(@salmon.parsed_data) raise "Object is not public" if object_can_be_public_and_it_is_not? - raise "Author does not match XML author" if author_does_not_match_xml_author? + raise Diaspora::AuthorXMLAuthorMismatch if author_does_not_match_xml_author? @object.save! if @object && @object.respond_to?(:save!) @object end diff --git a/lib/sidekiq_middlewares.rb b/lib/sidekiq_middlewares.rb new file mode 100644 index 000000000..c744c9105 --- /dev/null +++ b/lib/sidekiq_middlewares.rb @@ -0,0 +1,13 @@ +module SidekiqMiddlewares + class CleanAndShortBacktraces + def call(worker, item, queue) + yield + rescue Exception + backtrace = Rails.backtrace_cleaner.clean($!.backtrace) + backtrace.reject! { |line| line =~ /lib\/sidekiq_middlewares.rb/ } + limit = AppConfig.environment.sidekiq.backtrace.to_i + backtrace = [] if limit == 0 + raise $!, $!.message, backtrace[0..limit] + end + end +end diff --git a/lib/tasks/resque.rake b/lib/tasks/resque.rake deleted file mode 100644 index 773514d13..000000000 --- a/lib/tasks/resque.rake +++ /dev/null @@ -1,18 +0,0 @@ -require 'resque/tasks' - -task "resque:setup" do - require File.join(File.dirname(__FILE__), '..', '..', 'config', 'environment') - Rails.logger.info("event=resque_setup rails_env=#{Rails.env}") - - Resque.after_fork = Proc.new { ActiveRecord::Base.establish_connection } -end - -desc "Alias for resque:work (To run workers on Heroku)" -task "jobs:work" => "resque:work" - -desc 'clear your failure queue in resque. good for crons.' -task 'resque:clear_failed' => [:environment] do - puts "clearing resque failures" - Resque::Failure.clear - puts "complete!" -end diff --git a/lib/webfinger.rb b/lib/webfinger.rb index c70cb7469..93e2cf5f6 100644 --- a/lib/webfinger.rb +++ b/lib/webfinger.rb @@ -19,7 +19,7 @@ class Webfinger end def self.in_background(account, opts={}) - Resque.enqueue(Jobs::FetchWebfinger, account) + Workers::FetchWebfinger.perform_async(account) end #everything below should be private I guess diff --git a/script/ci/build.sh b/script/ci/build.sh index 32b015df4..127874287 100755 --- a/script/ci/build.sh +++ b/script/ci/build.sh @@ -5,7 +5,7 @@ echo "Setting up database.yml for $DB" cp config/database.yml.example config/database.yml if [ "$DB" = "postgres" ]; then - sed -i 's/*choose/*postgres_travis/' config/database.yml + sed -i 's/*common/*postgres_travis/' config/database.yml fi command="bundle exec rake --trace ci:travis:${BUILD_TYPE}" diff --git a/script/server b/script/server index e31b4de5b..4a551f6a3 100755 --- a/script/server +++ b/script/server @@ -35,9 +35,8 @@ os=`uname -s` eval $(bundle exec ruby ./script/get_config.rb \ port=server.port \ db=server.database \ - workers=server.resque_workers \ single_process_mode=environment.single_process_mode? - embed_resque_worker=server.embed_resque_worker + embed_sidekiq_worker=server.embed_sidekiq_worker ) if [ -z "$DB" ]; then @@ -45,10 +44,6 @@ if [ -z "$DB" ]; then export DB fi -if [ "$single_process_mode" = "true" -o "$embed_resque_worker" = "true" ]; then - workers=0 -fi - args="$@" prev_arg='' for arg in $( echo $args | awk '{ for (i = 1; i <= NF; i++) print $i}') @@ -106,13 +101,11 @@ fi # Start Diaspora echo -n "Starting Diaspora in $RAILS_ENV mode on port $port " -if [ "$embed_resque_worker" = "true" ]; then - echo "with a resque worker embeded into unicorn." +if [ "$embed_sidekiq_worker" = "true" ]; then + echo "with a sidekiq worker embeded into unicorn." elif [ "$single_process_mode" = "true" ]; then echo "with job processing inside the request cycle." -else - echo "with $workers resque workers." fi echo "" -exec bundle exec foreman start -m "web=1,worker=$workers" -p $port +exec bundle exec foreman start -m "web=1,sidekiq=1" -p $port diff --git a/spec/controllers/devise/passwords_controller_spec.rb b/spec/controllers/devise/passwords_controller_spec.rb index b9c61ae9f..5dbdbc786 100644 --- a/spec/controllers/devise/passwords_controller_spec.rb +++ b/spec/controllers/devise/passwords_controller_spec.rb @@ -17,8 +17,9 @@ describe Devise::PasswordsController do post :create, "user" => {"email" => "foo@example.com"} response.should be_success end + it "doesn't send email" do - Resque.should_not_receive(:enqueue) + Workers::ResetPassword.should_not_receive(:perform_async) post :create, "user" => {"email" => "foo@example.com"} end end @@ -27,10 +28,10 @@ describe Devise::PasswordsController do post :create, "user" => {"email" => alice.email} response.should redirect_to(new_user_session_path) end - it "sends email (enqueued to Resque)" do - Resque.should_receive(:enqueue).with(Jobs::ResetPassword, alice.id) + it "sends email (enqueued to Sidekiq)" do + Workers::ResetPassword.should_receive(:perform_async).with(alice.id) post :create, "user" => {"email" => alice.email} end end end -end \ No newline at end of file +end diff --git a/spec/controllers/publics_controller_spec.rb b/spec/controllers/publics_controller_spec.rb index e2075767b..16df77950 100644 --- a/spec/controllers/publics_controller_spec.rb +++ b/spec/controllers/publics_controller_spec.rb @@ -33,7 +33,7 @@ describe PublicsController do it 'enqueues a ReceiveUnencryptedSalmon job' do xml = "stuff" - Resque.should_receive(:enqueue).with(Jobs::ReceiveUnencryptedSalmon, xml) + Workers::ReceiveUnencryptedSalmon.should_receive(:perform_async).with(xml) post :receive_public, :xml => xml end end @@ -47,7 +47,7 @@ describe PublicsController do end it 'enqueues a receive job' do - Resque.should_receive(:enqueue).with(Jobs::ReceiveEncryptedSalmon, @user.id, xml).once + Workers::ReceiveEncryptedSalmon.should_receive(:perform_async).with(@user.id, xml).once post :receive, "guid" => @user.person.guid.to_s, "xml" => xml end @@ -60,7 +60,7 @@ describe PublicsController do salmon_factory = Salmon::EncryptedSlap.create_by_user_and_activity(@user, xml2) enc_xml = salmon_factory.xml_for(user2.person) - Resque.should_receive(:enqueue).with(Jobs::ReceiveEncryptedSalmon, @user.id, enc_xml).once + Workers::ReceiveEncryptedSalmon.should_receive(:perform_async).with(@user.id, enc_xml).once post :receive, "guid" => @user.person.guid.to_s, "xml" => CGI::escape(enc_xml) end diff --git a/spec/controllers/services_controller_spec.rb b/spec/controllers/services_controller_spec.rb index 8ce5d8fde..3446b1730 100644 --- a/spec/controllers/services_controller_spec.rb +++ b/spec/controllers/services_controller_spec.rb @@ -76,7 +76,7 @@ describe ServicesController do profile[:image_url] = "/non/default/image.jpg" profile.save - Resque.should_not_receive(:enqueue) + Workers::FetchProfilePhoto.should_not_receive(:perform_async) post :create, :provider => 'twitter' end @@ -86,7 +86,7 @@ describe ServicesController do profile[:image_url] = nil profile.save - Resque.should_receive(:enqueue).with(Jobs::FetchProfilePhoto, @user.id, anything(), "https://service.com/fallback_lowres.jpg") + Workers::FetchProfilePhoto.should_receive(:perform_async).with(@user.id, anything(), "https://service.com/fallback_lowres.jpg") post :create, :provider => 'twitter' end diff --git a/spec/controllers/status_messages_controller_spec.rb b/spec/controllers/status_messages_controller_spec.rb index 57121d1c5..e78aa1f63 100644 --- a/spec/controllers/status_messages_controller_spec.rb +++ b/spec/controllers/status_messages_controller_spec.rb @@ -202,11 +202,12 @@ describe StatusMessagesController do end it "sets the pending bit of referenced photos" do - fantasy_resque do + inlined_jobs do post :create, @hash - @photo1.reload.pending.should be_false - @photo2.reload.pending.should be_false end + + @photo1.reload.pending.should be_false + @photo2.reload.pending.should be_false end end end diff --git a/spec/controllers/users_controller_spec.rb b/spec/controllers/users_controller_spec.rb index b72088692..bc2cc59fe 100644 --- a/spec/controllers/users_controller_spec.rb +++ b/spec/controllers/users_controller_spec.rb @@ -121,10 +121,6 @@ describe UsersController do end describe 'email' do - before do - Resque.stub!(:enqueue) - end - it 'disallow the user to change his new (unconfirmed) mail when it is the same as the old' do @user.email = "my@newemail.com" put(:update, :id => @user.id, :user => { :email => "my@newemail.com"}) @@ -157,7 +153,7 @@ describe UsersController do end it 'sends out activation email on success' do - Resque.should_receive(:enqueue).with(Jobs::Mail::ConfirmEmail, @user.id).once + Workers::Mail::ConfirmEmail.should_receive(:perform_async).with(@user.id).once put(:update, :id => @user.id, :user => { :email => "my@newemail.com"}) end end @@ -209,7 +205,7 @@ describe UsersController do describe '#destroy' do it 'does nothing if the password does not match' do - Resque.should_not_receive(:enqueue) + Workers::DeleteAccount.should_not_receive(:perform_async) delete :destroy, :user => { :current_password => "stuff" } end @@ -219,7 +215,7 @@ describe UsersController do end it 'enqueues a delete job' do - Resque.should_receive(:enqueue).with(Jobs::DeleteAccount, anything) + Workers::DeleteAccount.should_receive(:perform_async).with(anything) delete :destroy, :user => { :current_password => "bluepin7" } end end diff --git a/spec/integration/attack_vectors_spec.rb b/spec/integration/attack_vectors_spec.rb index f778182a1..398297428 100644 --- a/spec/integration/attack_vectors_spec.rb +++ b/spec/integration/attack_vectors_spec.rb @@ -85,9 +85,9 @@ describe "attack vectors" do zord = Postzord::Receiver::Private.new(bob, :salmon_xml => salmon_xml) expect { - expect_error /Contact required/ do + expect { zord.perform! - end + }.to raise_error Diaspora::ContactRequiredUnlessRequest }.to_not change(Post, :count) user_should_not_see_guid(bob, bad_post_guid) @@ -110,9 +110,9 @@ describe "attack vectors" do #bob sends it to himself????? zord = Postzord::Receiver::Private.new(bob, :salmon_xml => salmon_xml) - expect_error /Contact required/ do + expect { zord.perform! - end + }.to raise_error Diaspora::ContactRequiredUnlessRequest #alice still should not see eves original post, even though bob sent it to her user_should_not_see_guid(alice, original_message.guid) @@ -125,9 +125,9 @@ describe "attack vectors" do profile.first_name = "Not BOB" expect { - expect_error /Author does not match XML author/ do + expect { receive(profile, :from => alice, :by => bob) - end + }.to raise_error Diaspora::AuthorXMLAuthorMismatch }.to_not change(eve.profile, :first_name) end end @@ -135,9 +135,9 @@ describe "attack vectors" do it 'public stuff should not be spoofed from another author' do post = FactoryGirl.build(:status_message, :public => true, :author => eve.person) - expect_error /Author does not match XML author/ do + expect { receive_public(post, :from => alice) - end + }.to raise_error Diaspora::AuthorXMLAuthorMismatch end end @@ -209,9 +209,9 @@ describe "attack vectors" do end expect { - expect_error /Author does not match XML author/ do + expect { receive(retraction, :from => alice, :by => bob) - end + }.to raise_error Diaspora::AuthorXMLAuthorMismatch }.to_not change(bob.visible_shareables(Post), :count) end @@ -239,10 +239,10 @@ describe "attack vectors" do end expect{ - expect_error /Author does not match XML author/ do + expect { receive(retraction, :from => alice, :by => bob) - end - }.to_not change(bob.contacts, :count) + }.to raise_error Diaspora::AuthorXMLAuthorMismatch + }.to_not change(bob.contacts, :count) end it 'does not let another user update other persons post' do diff --git a/spec/integration/dispatching_spec.rb b/spec/integration/dispatching_spec.rb index 07422338b..76a275f86 100644 --- a/spec/integration/dispatching_spec.rb +++ b/spec/integration/dispatching_spec.rb @@ -7,13 +7,14 @@ describe "Dispatching" do # Luke has a public post and comments on it post = FactoryGirl.create(:status_message, :public => true, :author => luke.person) - fantasy_resque do - comment = luke.comment!(post, "awesomesauseum") + comment = luke.comment!(post, "awesomesauseum") + + inlined_jobs do # Luke now retracts his comment Postzord::Dispatcher::Public.should_not_receive(:new) Postzord::Dispatcher::Private.should_receive(:new).and_return(stub(:post => true)) luke.retract(comment) - end + end end end end diff --git a/spec/integration/receiving_spec.rb b/spec/integration/receiving_spec.rb index b58d79f03..bc9b3c765 100644 --- a/spec/integration/receiving_spec.rb +++ b/spec/integration/receiving_spec.rb @@ -42,11 +42,12 @@ describe 'a user receives a post' do end it "should show bob's post to alice" do - fantasy_resque do + inlined_jobs do |queue| sm = bob.build_post(:status_message, :text => "hi") sm.save! bob.aspects.reload bob.add_to_streams(sm, [@bobs_aspect]) + queue.drain_all bob.dispatch_post(sm, :to => @bobs_aspect) end @@ -173,7 +174,7 @@ describe 'a user receives a post' do context 'remote' do before do - fantasy_resque do + inlined_jobs do |queue| connect_users(alice, @alices_aspect, eve, @eves_aspect) @post = alice.post(:status_message, :text => "hello", :to => @alices_aspect.id) @@ -183,6 +184,7 @@ describe 'a user receives a post' do receive_with_zord(eve, alice.person, xml) comment = eve.comment!(@post, 'tada') + queue.drain_all # After Eve creates her comment, it gets sent to Alice, who signs it with her private key # before relaying it out to the contacts on the top-level post comment.parent_author_signature = comment.sign_with_key(alice.encryption_key) @@ -190,6 +192,7 @@ describe 'a user receives a post' do comment.delete comment_with_whitespace = alice.comment!(@post, ' I cannot lift my thumb from the spacebar ') + queue.drain_all @xml_with_whitespace = comment_with_whitespace.to_diaspora_xml @guid_with_whitespace = comment_with_whitespace.guid comment_with_whitespace.delete @@ -253,13 +256,13 @@ describe 'a user receives a post' do end it 'does not raise a `Mysql2::Error: Duplicate entry...` exception on save' do - fantasy_resque do + inlined_jobs do @comment = bob.comment!(@post, 'tada') @xml = @comment.to_diaspora_xml - - lambda { + + expect { receive_with_zord(alice, bob.person, @xml) - }.should_not raise_exception + }.to_not raise_exception end end end diff --git a/spec/lib/configuration_methods_spec.rb b/spec/lib/configuration_methods_spec.rb index 34bbc7065..a87038de2 100644 --- a/spec/lib/configuration_methods_spec.rb +++ b/spec/lib/configuration_methods_spec.rb @@ -95,14 +95,14 @@ describe Configuration::Methods do end end - describe "#get_redis_instance" do + describe "#get_redis_options" do context "with REDISTOGO_URL set" do before do ENV["REDISTOGO_URL"] = "redis://myserver" end it "uses that" do - @settings.get_redis_instance.client.host.should == "myserver" + @settings.get_redis_options[:url].should match "myserver" end end @@ -113,7 +113,7 @@ describe Configuration::Methods do end it "uses that" do - @settings.get_redis_instance.client.host.should == "yourserver" + @settings.get_redis_options[:url].should match "yourserver" end end @@ -125,19 +125,7 @@ describe Configuration::Methods do end it "uses that" do - @settings.get_redis_instance.client.host.should == "ourserver" - end - end - - context "with nothing set" do - before do - @settings.environment.redis = nil - ENV["REDISTOGO_URL"] = nil - ENV["REDIS_URL"] = nil - end - - it "uses localhost" do - @settings.get_redis_instance.client.host.should == "127.0.0.1" + @settings.get_redis_options[:url].should match "ourserver" end end @@ -149,7 +137,26 @@ describe Configuration::Methods do end it "uses that" do - @settings.get_redis_instance.client.path.should == "/tmp/redis.sock" + @settings.get_redis_options[:url].should match "/tmp/redis.sock" + end + end + end + + describe "sidekiq_log" do + context "with a relative log set" do + it "joins that with Rails.root" do + path = "/some/path/" + Rails.stub!(:root).and_return(stub(join: path)) + @settings.environment.sidekiq.log = "relative_path" + @settings.sidekiq_log.should match path + end + end + + context "with a absolute path" do + it "just returns that" do + path = "/foobar.log" + @settings.environment.sidekiq.log = path + @settings.sidekiq_log.should == path end end end diff --git a/spec/lib/postzord/dispatcher_spec.rb b/spec/lib/postzord/dispatcher_spec.rb index 4e39a1d95..23af27bf0 100644 --- a/spec/lib/postzord/dispatcher_spec.rb +++ b/spec/lib/postzord/dispatcher_spec.rb @@ -209,7 +209,7 @@ describe Postzord::Dispatcher do it 'should queue an HttpMultiJob for the remote people' do Postzord::Dispatcher::Public.any_instance.unstub(:deliver_to_remote) - Resque.should_receive(:enqueue).with(Jobs::HttpMulti, alice.id, anything, @remote_people.map{|p| p.id}, anything).once + Workers::HttpMulti.should_receive(:perform_async).with(alice.id, anything, @remote_people.map{|p| p.id}, anything).once @mailman.send(:deliver_to_remote, @remote_people) Postzord::Dispatcher::Public.stub(:deliver_to_remote) @@ -224,18 +224,18 @@ describe Postzord::Dispatcher do it 'queues a batch receive' do local_people = [] local_people << alice.person - Resque.should_receive(:enqueue).with(Jobs::ReceiveLocalBatch, @sm.class.to_s, @sm.id, [alice.id]).once + Workers::ReceiveLocalBatch.should_receive(:perform_async).with(@sm.class.to_s, @sm.id, [alice.id]).once @mailman.send(:deliver_to_local, local_people) end it 'returns if people are empty' do - Resque.should_not_receive(:enqueue) + Workers::ReceiveLocalBatch.should_not_receive(:perform_async) @mailman.send(:deliver_to_local, []) end it 'returns if the object is a profile' do @mailman.instance_variable_set(:@object, Profile.new) - Resque.should_not_receive(:enqueue) + Workers::ReceiveLocalBatch.should_not_receive(:perform_async) @mailman.send(:deliver_to_local, [1]) end end @@ -277,8 +277,8 @@ describe Postzord::Dispatcher do end it 'queues a job to notify the hub' do - Resque.stub!(:enqueue).with(Jobs::PostToService, anything, anything, anything) - Resque.should_receive(:enqueue).with(Jobs::PublishToHub, alice.public_url) + Workers::PostToService.stub!(:perform_async).with(anything, anything, anything) + Workers::PublishToHub.should_receive(:perform_async).with(alice.public_url) @zord.send(:deliver_to_services, nil, []) end @@ -297,17 +297,17 @@ describe Postzord::Dispatcher do alice.services << @s2 mailman = Postzord::Dispatcher.build(alice, FactoryGirl.create(:status_message), :url => "http://joindiaspora.com/p/123", :services => [@s1]) - Resque.stub!(:enqueue).with(Jobs::PublishToHub, anything) - Resque.stub!(:enqueue).with(Jobs::HttpMulti, anything, anything, anything) - Resque.should_receive(:enqueue).with(Jobs::PostToService, @s1.id, anything, anything) + Workers::PublishToHub.stub!(:perform_async).with(anything) + Workers::HttpMulti.stub!(:perform_async).with(anything, anything, anything) + Workers::PostToService.should_receive(:perform_async).with(@s1.id, anything, anything) mailman.post end it 'does not push to services if none are specified' do mailman = Postzord::Dispatcher.build(alice, FactoryGirl.create(:status_message), :url => "http://joindiaspora.com/p/123") - Resque.stub!(:enqueue).with(Jobs::PublishToHub, anything) - Resque.should_not_receive(:enqueue).with(Jobs::PostToService, anything, anything, anything) + Workers::PublishToHub.stub!(:perform_async).with(anything) + Workers::PostToService.should_not_receive(:perform_async).with(anything, anything, anything) mailman.post end @@ -315,7 +315,7 @@ describe Postzord::Dispatcher do retraction = SignedRetraction.build(alice, FactoryGirl.create(:status_message)) mailman = Postzord::Dispatcher.build(alice, retraction, :url => "http://joindiaspora.com/p/123", :services => [@service]) - Resque.should_receive(:enqueue).with(Jobs::DeletePostFromService, anything, anything) + Workers::DeletePostFromService.should_receive(:perform_async).with(anything, anything) mailman.post end @@ -324,7 +324,7 @@ describe Postzord::Dispatcher do service = Services::Twitter.new(access_token: "nope") mailman = Postzord::Dispatcher.build(alice, retraction, :url => "http://joindiaspora.com/p/123", :services => [service]) - Resque.should_not_receive(:enqueue).with(Jobs::DeletePostFromService, anything, anything) + Workers::DeletePostFromService.should_not_receive(:perform_async).with(anything, anything) mailman.post end end @@ -338,7 +338,7 @@ describe Postzord::Dispatcher do describe '#notify_users' do it 'enqueues a NotifyLocalUsers job' do - Resque.should_receive(:enqueue).with(Jobs::NotifyLocalUsers, [bob.id], @zord.object.class.to_s, @zord.object.id, @zord.object.author.id) + Workers::NotifyLocalUsers.should_receive(:perform_async).with([bob.id], @zord.object.class.to_s, @zord.object.id, @zord.object.author.id) @zord.send(:notify_users, [bob]) end end diff --git a/spec/lib/postzord/receiver/public_spec.rb b/spec/lib/postzord/receiver/public_spec.rb index 766686677..e94886dcc 100644 --- a/spec/lib/postzord/receiver/public_spec.rb +++ b/spec/lib/postzord/receiver/public_spec.rb @@ -61,13 +61,13 @@ describe Postzord::Receiver::Public do @receiver.perform! end - it 'enqueues a Jobs::ReceiveLocalBatch' do - Resque.should_receive(:enqueue).with(Jobs::ReceiveLocalBatch, anything, anything, anything) + it 'enqueues a Workers::ReceiveLocalBatch' do + Workers::ReceiveLocalBatch.should_receive(:perform_async).with(anything, anything, anything) @receiver.perform! end it 'intergrates' do - fantasy_resque do + inlined_jobs do @receiver.perform! end end diff --git a/spec/lib/webfinger_spec.rb b/spec/lib/webfinger_spec.rb index 34cac08f9..7d247814d 100644 --- a/spec/lib/webfinger_spec.rb +++ b/spec/lib/webfinger_spec.rb @@ -31,8 +31,8 @@ describe Webfinger do end describe '.in_background' do - it 'enqueues a Jobs::FetchWebfinger job' do - Resque.should_receive(:enqueue).with(Jobs::FetchWebfinger, account) + it 'enqueues a Workers::FetchWebfinger job' do + Workers::FetchWebfinger.should_receive(:perform_async).with(account) Webfinger.in_background(account) end end diff --git a/spec/models/account_deletion_spec.rb b/spec/models/account_deletion_spec.rb index 793a33b98..0b97b6db9 100644 --- a/spec/models/account_deletion_spec.rb +++ b/spec/models/account_deletion_spec.rb @@ -10,8 +10,8 @@ describe AccountDeletion do a.diaspora_handle.should == alice.person.diaspora_handle end - it 'fires a resque job after creation'do - Resque.should_receive(:enqueue).with(Jobs::DeleteAccount, anything) + it 'fires a job after creation'do + Workers::DeleteAccount.should_receive(:perform_async).with(anything) AccountDeletion.create(:person => alice.person) end diff --git a/spec/models/photo_spec.rb b/spec/models/photo_spec.rb index faf9350f7..b168e10c0 100644 --- a/spec/models/photo_spec.rb +++ b/spec/models/photo_spec.rb @@ -197,7 +197,7 @@ describe Photo do describe 'remote photos' do before do - Jobs::ProcessPhoto.perform(@saved_photo.id) + Workers::ProcessPhoto.new.perform(@saved_photo.id) end it 'should set the remote_photo on marshalling' do @@ -229,8 +229,8 @@ describe Photo do end describe '#queue_processing_job' do - it 'should queue a resque job to process the images' do - Resque.should_receive(:enqueue).with(Jobs::ProcessPhoto, @photo.id) + it 'should queue a job to process the images' do + Workers::ProcessPhoto.should_receive(:perform_async).with(@photo.id) @photo.queue_processing_job end end diff --git a/spec/models/status_message_spec.rb b/spec/models/status_message_spec.rb index 9fb1700f6..14f3d7da2 100644 --- a/spec/models/status_message_spec.rb +++ b/spec/models/status_message_spec.rb @@ -339,7 +339,7 @@ STR it 'should queue a GatherOembedData if it includes a link' do sm = FactoryGirl.build(:status_message, :text => @message_text) - Resque.should_receive(:enqueue).with(Jobs::GatherOEmbedData, instance_of(Fixnum), instance_of(String)) + Workers::GatherOEmbedData.should_receive(:perform_async).with(instance_of(Fixnum), instance_of(String)) sm.save end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index b0d760da0..58fcbc142 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -587,22 +587,22 @@ describe User do alice.disable_mail = false alice.save - Resque.should_receive(:enqueue).with(Jobs::Mail::StartedSharing, alice.id, 'contactrequestid').once - alice.mail(Jobs::Mail::StartedSharing, alice.id, 'contactrequestid') + Workers::Mail::StartedSharing.should_receive(:perform_async).with(alice.id, 'contactrequestid').once + alice.mail(Workers::Mail::StartedSharing, alice.id, 'contactrequestid') end - it 'does not enqueue a mail job if the correct corresponding job has a prefrence entry' do + it 'does not enqueue a mail job if the correct corresponding job has a preference entry' do alice.user_preferences.create(:email_type => 'started_sharing') - Resque.should_not_receive(:enqueue) - alice.mail(Jobs::Mail::StartedSharing, alice.id, 'contactrequestid') + Workers::Mail::StartedSharing.should_not_receive(:perform_async) + alice.mail(Workers::Mail::StartedSharing, alice.id, 'contactrequestid') end it 'does not send a mail if disable_mail is set to true' do alice.disable_mail = true alice.save alice.reload - Resque.should_not_receive(:enqueue) - alice.mail(Jobs::Mail::StartedSharing, alice.id, 'contactrequestid') + Workers::Mail::StartedSharing.should_not_receive(:perform_async) + alice.mail(Workers::Mail::StartedSharing, alice.id, 'contactrequestid') end end @@ -721,12 +721,12 @@ describe User do describe '#mail_confirm_email' do it 'enqueues a mail job on user with unconfirmed email' do user.update_attribute(:unconfirmed_email, "alice@newmail.com") - Resque.should_receive(:enqueue).with(Jobs::Mail::ConfirmEmail, alice.id).once + Workers::Mail::ConfirmEmail.should_receive(:perform_async).with(alice.id).once alice.mail_confirm_email.should eql(true) end it 'enqueues NO mail job on user without unconfirmed email' do - Resque.should_not_receive(:enqueue).with(Jobs::Mail::ConfirmEmail, alice.id) + Workers::Mail::ConfirmEmail.should_not_receive(:perform_async).with(alice.id) alice.mail_confirm_email.should eql(false) end end @@ -843,7 +843,7 @@ describe User do it "queues up a job to send the reset password instructions" do user = FactoryGirl.create :user - Resque.should_receive(:enqueue).with(Jobs::ResetPassword, user.id) + Workers::ResetPassword.should_receive(:perform_async).with(user.id) user.send_reset_password_instructions end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 337a8de0d..ceef36105 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -19,6 +19,7 @@ prefork = proc do require 'rspec/rails' require 'webmock/rspec' require 'factory_girl' + require 'sidekiq/testing' include HelperMethods diff --git a/spec/support/fake_resque.rb b/spec/support/fake_resque.rb deleted file mode 100644 index e08eb0a03..000000000 --- a/spec/support/fake_resque.rb +++ /dev/null @@ -1,19 +0,0 @@ -module Resque - def enqueue(klass, *args) - if $process_queue - klass.send(:perform, *args) - else - true - end - end -end - -module HelperMethods - def fantasy_resque - former_value = $process_queue - $process_queue = true - result = yield - $process_queue = former_value - result - end -end diff --git a/spec/support/fake_typhoeus.rb b/spec/support/fake_typhoeus.rb index 8742a8828..fdbacaba4 100644 --- a/spec/support/fake_typhoeus.rb +++ b/spec/support/fake_typhoeus.rb @@ -10,13 +10,13 @@ end def disable_typhoeus silence_warnings do - Jobs::HttpMulti.const_set('Hydra', FakeHydra) - Jobs::HttpMulti.const_set('Request', FakeHydraRequest) + Workers::HttpMulti.const_set('Hydra', FakeHydra) + Workers::HttpMulti.const_set('Request', FakeHydraRequest) end end def enable_typhoeus silence_warnings do - Jobs::HttpMulti.const_set('Hydra', Typhoeus::Hydra) - Jobs::HttpMulti.const_set('Request', Typhoeus::Request) + Workers::HttpMulti.const_set('Hydra', Typhoeus::Hydra) + Workers::HttpMulti.const_set('Request', Typhoeus::Request) end end diff --git a/spec/support/inlined_jobs.rb b/spec/support/inlined_jobs.rb new file mode 100644 index 000000000..e7dcabc47 --- /dev/null +++ b/spec/support/inlined_jobs.rb @@ -0,0 +1,10 @@ +module HelperMethods + def inlined_jobs + Sidekiq::Worker.clear_all + result = yield Sidekiq::Worker + Sidekiq::Worker.drain_all + result + rescue NoMethodError + yield Sidekiq::Worker if block_given? # Never error out on our own + end +end diff --git a/spec/support/user_methods.rb b/spec/support/user_methods.rb index 74fd44c18..2e7b04089 100644 --- a/spec/support/user_methods.rb +++ b/spec/support/user_methods.rb @@ -7,13 +7,13 @@ class User alias_method :share_with_original, :share_with def share_with(*args) - fantasy_resque do + inlined_jobs do share_with_original(*args) end end def post(class_name, opts = {}) - fantasy_resque do + inlined_jobs do p = build_post(class_name, opts) if p.save! self.aspects.reload diff --git a/spec/models/jobs/delete_account_spec.rb b/spec/workers/delete_account_spec.rb similarity index 84% rename from spec/models/jobs/delete_account_spec.rb rename to spec/workers/delete_account_spec.rb index 507f0d2b9..fd6c98a6b 100644 --- a/spec/models/jobs/delete_account_spec.rb +++ b/spec/workers/delete_account_spec.rb @@ -4,14 +4,14 @@ require 'spec_helper' -describe Jobs::DeleteAccount do +describe Workers::DeleteAccount do describe '#perform' do it 'performs the account deletion' do account_deletion = stub AccountDeletion.stub(:find).and_return(account_deletion) account_deletion.should_receive(:perform!) - Jobs::DeleteAccount.perform(1) + Workers::DeleteAccount.new.perform(1) end end end diff --git a/spec/models/jobs/delete_post_from_service_spec.rb b/spec/workers/delete_post_from_service_spec.rb similarity index 76% rename from spec/models/jobs/delete_post_from_service_spec.rb rename to spec/workers/delete_post_from_service_spec.rb index 397fcbcdb..0860308f1 100644 --- a/spec/models/jobs/delete_post_from_service_spec.rb +++ b/spec/workers/delete_post_from_service_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Jobs::DeletePostFromService do +describe Workers::DeletePostFromService do before do @user = alice @post = @user.post(:status_message, :text => "hello", :to =>@user.aspects.first.id, :public =>true, :facebook_id => "23456" ) @@ -11,6 +11,6 @@ describe Jobs::DeletePostFromService do url = "foobar" m.should_receive(:delete_post) Service.stub!(:find_by_id).and_return(m) - Jobs::DeletePostFromService.perform("123", @post.id.to_s) + Workers::DeletePostFromService.new.perform("123", @post.id.to_s) end end diff --git a/spec/models/jobs/fetch_profile_photo_spec.rb b/spec/workers/fetch_profile_photo_spec.rb similarity index 84% rename from spec/models/jobs/fetch_profile_photo_spec.rb rename to spec/workers/fetch_profile_photo_spec.rb index e82c83c28..6e4afcfb2 100644 --- a/spec/models/jobs/fetch_profile_photo_spec.rb +++ b/spec/workers/fetch_profile_photo_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Jobs::FetchProfilePhoto do +describe Workers::FetchProfilePhoto do before do @user = alice @service = FactoryGirl.build(:service, :user => alice) @@ -22,7 +22,7 @@ describe Jobs::FetchProfilePhoto do @photo_stub.should_receive(:save!).and_return(true) Photo.should_receive(:diaspora_initialize).with(hash_including(:author => @user.person, :image_url => @url, :pending => true)).and_return(@photo_stub) - Jobs::FetchProfilePhoto.perform(@user.id, @service.id) + Workers::FetchProfilePhoto.new.perform(@user.id, @service.id) end context "service does not have a profile_photo_url" do @@ -30,7 +30,7 @@ describe Jobs::FetchProfilePhoto do @service.stub!(:profile_photo_url).and_return(nil) Photo.should_not_receive(:diaspora_initialize) - Jobs::FetchProfilePhoto.perform(@user.id, @service.id) + Workers::FetchProfilePhoto.new.perform(@user.id, @service.id) end it "fetches fallback if it's provided" do @@ -38,7 +38,7 @@ describe Jobs::FetchProfilePhoto do @service.stub!(:profile_photo_url).and_return(nil) Photo.should_receive(:diaspora_initialize).with(hash_including(:author => @user.person, :image_url => "https://service.com/fallback_lowres.jpg", :pending => true)).and_return(@photo_stub) - Jobs::FetchProfilePhoto.perform(@user.id, @service.id, "https://service.com/fallback_lowres.jpg") + Workers::FetchProfilePhoto.new.perform(@user.id, @service.id, "https://service.com/fallback_lowres.jpg") end end @@ -53,6 +53,6 @@ describe Jobs::FetchProfilePhoto do :image_url_small => "small.jpg" })) - Jobs::FetchProfilePhoto.perform(@user.id, @service.id) + Workers::FetchProfilePhoto.new.perform(@user.id, @service.id) end end diff --git a/spec/models/jobs/gather_o_embed_data_spec.rb b/spec/workers/gather_o_embed_data_spec.rb similarity index 75% rename from spec/models/jobs/gather_o_embed_data_spec.rb rename to spec/workers/gather_o_embed_data_spec.rb index 406cc31f6..9dc2710ac 100644 --- a/spec/models/jobs/gather_o_embed_data_spec.rb +++ b/spec/workers/gather_o_embed_data_spec.rb @@ -1,5 +1,5 @@ require 'spec_helper' -describe Jobs::GatherOEmbedData do +describe Workers::GatherOEmbedData do before do @flickr_oembed_data = { "trusted_endpoint_url" => "__!SPOOFED!__", @@ -30,34 +30,40 @@ describe Jobs::GatherOEmbedData do describe '.perform' do it 'requests not data from the internet' do - Jobs::GatherOEmbedData.perform(@status_message.id, @flickr_photo_url) + Workers::GatherOEmbedData.new.perform(@status_message.id, @flickr_photo_url) a_request(:get, @flickr_oembed_get_request).should have_been_made end it 'requests not data from the internet only once' do 2.times do |n| - Jobs::GatherOEmbedData.perform(@status_message.id, @flickr_photo_url) + Workers::GatherOEmbedData.new.perform(@status_message.id, @flickr_photo_url) end a_request(:get, @flickr_oembed_get_request).should have_been_made.times(1) end it 'creates one cache entry' do - Jobs::GatherOEmbedData.perform(@status_message.id, @flickr_photo_url) + Workers::GatherOEmbedData.new.perform(@status_message.id, @flickr_photo_url) expected_data = @flickr_oembed_data expected_data['trusted_endpoint_url'] = @flickr_oembed_url OEmbedCache.find_by_url(@flickr_photo_url).data.should == expected_data - Jobs::GatherOEmbedData.perform(@status_message.id, @flickr_photo_url) + Workers::GatherOEmbedData.new.perform(@status_message.id, @flickr_photo_url) OEmbedCache.count(:conditions => {:url => @flickr_photo_url}).should == 1 end it 'creates no cache entry for unsupported pages' do - Jobs::GatherOEmbedData.perform(@status_message.id, @no_oembed_url) + Workers::GatherOEmbedData.new.perform(@status_message.id, @no_oembed_url) OEmbedCache.find_by_url(@no_oembed_url).should be_nil end + + it 'gracefully handles a deleted post' do + expect { + Workers::GatherOEmbedData.new.perform(0, @flickr_photo_url) + }.to_not raise_error ActiveRecord::RecordNotFound + end end end diff --git a/spec/models/jobs/http_multi_spec.rb b/spec/workers/http_multi_spec.rb similarity index 77% rename from spec/models/jobs/http_multi_spec.rb rename to spec/workers/http_multi_spec.rb index 5556070a2..b111d8571 100644 --- a/spec/models/jobs/http_multi_spec.rb +++ b/spec/workers/http_multi_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Jobs::HttpMulti do +describe Workers::HttpMulti do before :all do WebMock.disable_net_connect!(:allow_localhost => true) enable_typhoeus @@ -29,7 +29,7 @@ describe Jobs::HttpMulti do Typhoeus::Hydra.stub!(:new).and_return(@hydra) people_ids = @people.map{ |p| p.id } - Jobs::HttpMulti.perform(bob.id, @post_xml, people_ids, "Postzord::Dispatcher::Private") + Workers::HttpMulti.new.perform(bob.id, @post_xml, people_ids, "Postzord::Dispatcher::Private") end it 'retries' do @@ -38,9 +38,9 @@ describe Jobs::HttpMulti do @hydra.stub(:post, person.receive_url).and_return(@failed_response) Typhoeus::Hydra.stub!(:new).and_return(@hydra) - - Resque.should_receive(:enqueue).with(Jobs::HttpMulti, bob.id, @post_xml, [person.id], anything, 1).once - Jobs::HttpMulti.perform(bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private") + + Workers::HttpMulti.should_receive(:perform_in).with(1.hour, bob.id, @post_xml, [person.id], anything, 1).once + Workers::HttpMulti.new.perform(bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private") end it 'max retries' do @@ -50,8 +50,8 @@ describe Jobs::HttpMulti do Typhoeus::Hydra.stub!(:new).and_return(@hydra) - Resque.should_not_receive(:enqueue) - Jobs::HttpMulti.perform(bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private", 3) + Workers::HttpMulti.should_not_receive(:perform_in) + Workers::HttpMulti.new.perform(bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private", 3) end it 'generates encrypted xml for people' do @@ -65,7 +65,7 @@ describe Jobs::HttpMulti do Salmon::EncryptedSlap.stub(:create_by_user_and_activity).and_return(salmon) salmon.should_receive(:xml_for).and_return("encrypted things") - Jobs::HttpMulti.perform(bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private") + Workers::HttpMulti.new.perform(bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private") end it 'updates http users who have moved to https' do @@ -79,7 +79,7 @@ describe Jobs::HttpMulti do Typhoeus::Hydra.stub!(:new).and_return(@hydra) - Jobs::HttpMulti.perform(bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private") + Workers::HttpMulti.new.perform(bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private") person.reload person.url.should == "https://remote.net/" end @@ -94,6 +94,6 @@ describe Jobs::HttpMulti do Typhoeus::Hydra.stub!(:new).and_return(@hydra) @hydra.should_receive(:queue).once - Jobs::HttpMulti.perform(bob.id, @post_xml, [@people[0].id, @people[1].id], "Postzord::Dispatcher::Private") + Workers::HttpMulti.new.perform(bob.id, @post_xml, [@people[0].id, @people[1].id], "Postzord::Dispatcher::Private") end end diff --git a/spec/models/jobs/mail/invite_user_by_email_spec.rb b/spec/workers/mail/invite_user_by_email_spec.rb similarity index 80% rename from spec/models/jobs/mail/invite_user_by_email_spec.rb rename to spec/workers/mail/invite_user_by_email_spec.rb index 2e4cdfcfe..4137361ad 100644 --- a/spec/models/jobs/mail/invite_user_by_email_spec.rb +++ b/spec/workers/mail/invite_user_by_email_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Jobs::Mail::InviteUserByEmail do +describe Workers::Mail::InviteUserByEmail do before do @sender = alice @email = 'bob@bob.com' @@ -12,6 +12,6 @@ describe Jobs::Mail::InviteUserByEmail do invitation = Invitation.create(:sender => @sender, :identifier => @email, :service => "email", :aspect => @aspect, :message => @message) invitation.should_receive(:send!) Invitation.stub(:find).and_return(invitation) - Jobs::Mail::InviteUserByEmail.perform(invitation.id) + Workers::Mail::InviteUserByEmail.new.perform(invitation.id) end end diff --git a/spec/models/jobs/mail/mentioned_spec.rb b/spec/workers/mail/mentioned_spec.rb similarity index 83% rename from spec/models/jobs/mail/mentioned_spec.rb rename to spec/workers/mail/mentioned_spec.rb index 54f454e1c..d643ad28e 100644 --- a/spec/models/jobs/mail/mentioned_spec.rb +++ b/spec/workers/mail/mentioned_spec.rb @@ -4,7 +4,7 @@ require 'spec_helper' -describe Jobs::Mail::Mentioned do +describe Workers::Mail::Mentioned do describe '#perfom' do it 'should call .deliver on the notifier object' do user = alice @@ -15,7 +15,7 @@ describe Jobs::Mail::Mentioned do mail_mock.should_receive(:deliver) Notifier.should_receive(:mentioned).with(user.id, sm.author.id, m.id).and_return(mail_mock) - Jobs::Mail::Mentioned.perform(user.id, sm.author.id, m.id) + Workers::Mail::Mentioned.new.perform(user.id, sm.author.id, m.id) end end end diff --git a/spec/models/jobs/mail/private_message_spec.rb b/spec/workers/mail/private_message_spec.rb similarity index 87% rename from spec/models/jobs/mail/private_message_spec.rb rename to spec/workers/mail/private_message_spec.rb index 3a74dd4de..36a50716b 100644 --- a/spec/models/jobs/mail/private_message_spec.rb +++ b/spec/workers/mail/private_message_spec.rb @@ -4,7 +4,7 @@ require 'spec_helper' -describe Jobs::Mail::PrivateMessage do +describe Workers::Mail::PrivateMessage do describe '#perfom_delegate' do it 'should call .deliver on the notifier object' do user1 = alice @@ -21,7 +21,7 @@ describe Jobs::Mail::PrivateMessage do mail_mock.should_receive(:deliver) Notifier.should_receive(:mentioned).with(user2.id, user1.person.id, message.id).and_return(mail_mock) - Jobs::Mail::Mentioned.perform(user2.id, user1.person.id, message.id) + Workers::Mail::Mentioned.new.perform(user2.id, user1.person.id, message.id) end end end diff --git a/spec/models/jobs/mail/reshared_spec.rb b/spec/workers/mail/reshared_spec.rb similarity index 84% rename from spec/models/jobs/mail/reshared_spec.rb rename to spec/workers/mail/reshared_spec.rb index 9c5e21afe..291682991 100644 --- a/spec/models/jobs/mail/reshared_spec.rb +++ b/spec/workers/mail/reshared_spec.rb @@ -4,7 +4,7 @@ require 'spec_helper' -describe Jobs::Mail::Reshared do +describe Workers::Mail::Reshared do describe '#perfom' do it 'should call .deliver on the notifier object' do sm = FactoryGirl.build(:status_message, :author => bob.person, :public => true) @@ -14,7 +14,7 @@ describe Jobs::Mail::Reshared do mail_mock.should_receive(:deliver) Notifier.should_receive(:reshared).with(bob.id, reshare.author.id, reshare.id).and_return(mail_mock) - Jobs::Mail::Reshared.perform(bob.id, reshare.author.id, reshare.id) + Workers::Mail::Reshared.new.perform(bob.id, reshare.author.id, reshare.id) end end end diff --git a/spec/models/jobs/notify_local_users_spec.rb b/spec/workers/notify_local_users_spec.rb similarity index 82% rename from spec/models/jobs/notify_local_users_spec.rb rename to spec/workers/notify_local_users_spec.rb index 982af5f1a..b058d0f77 100644 --- a/spec/models/jobs/notify_local_users_spec.rb +++ b/spec/workers/notify_local_users_spec.rb @@ -4,7 +4,7 @@ require 'spec_helper' -describe Jobs::NotifyLocalUsers do +describe Workers::NotifyLocalUsers do describe '#perfom' do it 'should call Notification.notify for each participant user' do person = FactoryGirl.create :person @@ -14,7 +14,7 @@ describe Jobs::NotifyLocalUsers do #User.should_receive(:where).and_return([alice, eve]) Notification.should_receive(:notify).with(instance_of(User), instance_of(StatusMessage), instance_of(Person)).twice - Jobs::NotifyLocalUsers.perform([alice.id, eve.id], post.class.to_s, post.id, person.id) + Workers::NotifyLocalUsers.new.perform([alice.id, eve.id], post.class.to_s, post.id, person.id) end end end diff --git a/spec/models/jobs/post_to_service_spec.rb b/spec/workers/post_to_service_spec.rb similarity index 80% rename from spec/models/jobs/post_to_service_spec.rb rename to spec/workers/post_to_service_spec.rb index 87cc46353..6f49a6f36 100644 --- a/spec/models/jobs/post_to_service_spec.rb +++ b/spec/workers/post_to_service_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Jobs::PostToService do +describe Workers::PostToService do it 'calls service#post with the given service' do user = alice aspect = user.aspects.create(:name => "yeah") @@ -10,6 +10,6 @@ describe Jobs::PostToService do url = "foobar" m.should_receive(:post).with(anything, url) Service.stub!(:find_by_id).and_return(m) - Jobs::PostToService.perform("123", post.id.to_s, url) + Workers::PostToService.new.perform("123", post.id.to_s, url) end end diff --git a/spec/models/jobs/process_photo_spec.rb b/spec/workers/process_photo_spec.rb similarity index 71% rename from spec/models/jobs/process_photo_spec.rb rename to spec/workers/process_photo_spec.rb index 1e1622343..5b07b4316 100644 --- a/spec/models/jobs/process_photo_spec.rb +++ b/spec/workers/process_photo_spec.rb @@ -1,11 +1,11 @@ require 'spec_helper' -describe Jobs::ProcessPhoto do +describe Workers::ProcessPhoto do before do @user = alice @aspect = @user.aspects.first - @fixture_name = File.join(File.dirname(__FILE__), '..', '..', 'fixtures', 'button.png') + @fixture_name = File.join(File.dirname(__FILE__), '..', 'fixtures', 'button.png') @saved_photo = @user.build_post(:photo, :user_file => File.open(@fixture_name), :to => @aspect.id) @saved_photo.save @@ -14,7 +14,7 @@ describe Jobs::ProcessPhoto do it 'saves the processed image' do @saved_photo.processed_image.path.should be_nil - result = Jobs::ProcessPhoto.perform(@saved_photo.id) + result = Workers::ProcessPhoto.new.perform(@saved_photo.id) @saved_photo.reload @@ -24,14 +24,14 @@ describe Jobs::ProcessPhoto do context 'when trying to process a photo that has already been processed' do before do - Jobs::ProcessPhoto.perform(@saved_photo.id) + Workers::ProcessPhoto.new.perform(@saved_photo.id) @saved_photo.reload end it 'does not process the photo' do processed_image_path = @saved_photo.processed_image.path - result = Jobs::ProcessPhoto.perform(@saved_photo.id) + result = Workers::ProcessPhoto.new.perform(@saved_photo.id) @saved_photo.reload @@ -42,13 +42,13 @@ describe Jobs::ProcessPhoto do context 'when a gif is uploaded' do before do - @fixture_name = File.join(File.dirname(__FILE__), '..', '..', 'fixtures', 'button.gif') + @fixture_name = File.join(File.dirname(__FILE__), '..', 'fixtures', 'button.gif') @saved_gif = @user.build_post(:photo, :user_file => File.open(@fixture_name), :to => @aspect.id) @saved_gif.save end it 'does not process the gif' do - result = Jobs::ProcessPhoto.perform(@saved_gif.id) + result = Workers::ProcessPhoto.new.perform(@saved_gif.id) @saved_gif.reload.processed_image.path.should be_nil result.should be false @@ -59,7 +59,7 @@ describe Jobs::ProcessPhoto do p = FactoryGirl.create(:remote_photo) p.unprocessed_image = nil expect{ - result = Jobs::ProcessPhoto.perform(p.id) + result = Workers::ProcessPhoto.new.perform(p.id) }.to_not raise_error end diff --git a/spec/models/jobs/publish_to_hub_spec.rb b/spec/workers/publish_to_hub_spec.rb similarity index 85% rename from spec/models/jobs/publish_to_hub_spec.rb rename to spec/workers/publish_to_hub_spec.rb index 27937cca5..571b2a884 100644 --- a/spec/models/jobs/publish_to_hub_spec.rb +++ b/spec/workers/publish_to_hub_spec.rb @@ -4,7 +4,7 @@ require 'spec_helper' -describe Jobs::PublishToHub do +describe Workers::PublishToHub do describe '.perform' do it 'calls pubsubhubbub' do url = "http://publiczone.com/" @@ -12,7 +12,7 @@ describe Jobs::PublishToHub do m.should_receive(:publish).with(url+'.atom') Pubsubhubbub.should_receive(:new).with(AppConfig.environment.pubsub_server).and_return(m) - Jobs::PublishToHub.perform(url) + Workers::PublishToHub.new.perform(url) end end end diff --git a/spec/models/jobs/receive_salmon_spec.rb b/spec/workers/receive_salmon_spec.rb similarity index 78% rename from spec/models/jobs/receive_salmon_spec.rb rename to spec/workers/receive_salmon_spec.rb index 43bf0c5bb..b3dcafb97 100644 --- a/spec/models/jobs/receive_salmon_spec.rb +++ b/spec/workers/receive_salmon_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Jobs::ReceiveEncryptedSalmon do +describe Workers::ReceiveEncryptedSalmon do before do @user = alice @xml = '' @@ -18,6 +18,6 @@ describe Jobs::ReceiveEncryptedSalmon do zord.should_receive(:perform!) Postzord::Receiver::Private.should_receive(:new).with(@user, hash_including(:salmon_xml => @xml)).and_return(zord) - Jobs::ReceiveEncryptedSalmon.perform(@user.id, @xml) + Workers::ReceiveEncryptedSalmon.new.perform(@user.id, @xml) end end diff --git a/spec/models/jobs/receive_spec.rb b/spec/workers/receive_spec.rb similarity index 83% rename from spec/models/jobs/receive_spec.rb rename to spec/workers/receive_spec.rb index 800287d9d..0972079c4 100644 --- a/spec/models/jobs/receive_spec.rb +++ b/spec/workers/receive_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Jobs::Receive do +describe Workers::Receive do before do @user = alice @person = FactoryGirl.create(:person) @@ -18,6 +18,6 @@ describe Jobs::Receive do zord_mock = mock() zord_mock.should_receive(:parse_and_receive).with(@xml) Postzord::Receiver::Private.should_receive(:new).with(@user, anything).and_return(zord_mock) - Jobs::Receive.perform(@user.id, @xml, @person.id) + Workers::Receive.new.perform(@user.id, @xml, @person.id) end end diff --git a/spec/models/jobs/resend_invitation_spec.rb b/spec/workers/resend_invitation_spec.rb similarity index 82% rename from spec/models/jobs/resend_invitation_spec.rb rename to spec/workers/resend_invitation_spec.rb index cd7d3508f..a0e0afd33 100644 --- a/spec/models/jobs/resend_invitation_spec.rb +++ b/spec/workers/resend_invitation_spec.rb @@ -4,14 +4,14 @@ require 'spec_helper' -describe Jobs::ResendInvitation do +describe Workers::ResendInvitation do describe '#perfom' do it 'should call .resend on the object' do invite = FactoryGirl.build(:invitation, :service => 'email', :identifier => 'foo@bar.com') Invitation.stub(:find).and_return(invite) invite.should_receive(:resend) - Jobs::ResendInvitation.perform(invite.id) + Workers::ResendInvitation.new.perform(invite.id) end end end diff --git a/spec/models/jobs/reset_password_spec.rb b/spec/workers/reset_password_spec.rb similarity index 79% rename from spec/models/jobs/reset_password_spec.rb rename to spec/workers/reset_password_spec.rb index 548d1fd4b..d1f09347f 100644 --- a/spec/models/jobs/reset_password_spec.rb +++ b/spec/workers/reset_password_spec.rb @@ -1,11 +1,11 @@ require 'spec_helper' -describe Jobs::ResetPassword do +describe Workers::ResetPassword do describe "#perform" do it "given a user id it sends the reset password instructions for that user" do user = FactoryGirl.create :user expect { - mail = Jobs::ResetPassword.perform(user.id) + mail = Workers::ResetPassword.new.perform(user.id) mail.to.should == [user.email] mail.body.should include("change your password") }.to change(Devise.mailer.deliveries, :length).by(1)