diff --git a/Gemfile b/Gemfile index 08482ea84..5975be402 100644 --- a/Gemfile +++ b/Gemfile @@ -71,6 +71,8 @@ gem 'ruby-oembed' gem 'resque', '1.10.0' gem 'resque-ensure-connected' gem 'resque-timeout', '1.0.0' +gem 'resque-scheduler' +gem 'resque-retry' gem 'SystemTimer', '1.2.1', :platforms => :ruby_18 # reporting diff --git a/Gemfile.lock b/Gemfile.lock index e34e0787b..27d9cf3da 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -381,6 +381,13 @@ GEM resque-ensure-connected (0.1.0) activerecord (>= 2.3.5) resque (~> 1.10.0) + resque-retry (0.1.0) + resque (>= 1.8.0) + resque-scheduler (>= 1.8.0) + resque-scheduler (1.9.9) + redis (>= 2.0.1) + resque (>= 1.8.0) + rufus-scheduler resque-timeout (1.0.0) resque (~> 1.0) rest-client (1.6.1) @@ -422,6 +429,8 @@ GEM archive-tar-minitar (>= 0.5.2) rubyntlm (0.1.1) rubyzip (0.9.4) + rufus-scheduler (2.0.11) + tzinfo (>= 0.3.23) sass (3.1.7) selenium-webdriver (2.7.0) childprocess (>= 0.2.1) @@ -533,6 +542,8 @@ DEPENDENCIES redcarpet (= 2.0.0b5) resque (= 1.10.0) resque-ensure-connected + resque-retry + resque-scheduler resque-timeout (= 1.0.0) rest-client (= 1.6.1) roxml! diff --git a/app/models/jobs/http_multi.rb b/app/models/jobs/http_multi.rb index ae9f8d47f..a17d5cce0 100644 --- a/app/models/jobs/http_multi.rb +++ b/app/models/jobs/http_multi.rb @@ -3,16 +3,30 @@ # the COPYRIGHT file. require 'uri' +require 'resque-retry' require File.join(Rails.root, 'lib/hydra_wrapper') module Jobs class HttpMulti < Base + extend Resque::Plugins::ExponentialBackoff @queue = :http + @backoff_strategy = [10.seconds, + 1.minute, + 10.minutes, + 1.hour, + 3.hours, + 6.hours, + 12.hours, + 1.day, + 2.days] - MAX_RETRIES = 3 + def self.args_for_retry(user_id, encoded_object_xml, person_ids, dispatcher_class_as_string) + [user_id, encoded_object_xml, @failed_people, dispatcher_class_as_string] + end - def self.perform(user_id, encoded_object_xml, person_ids, dispatcher_class_as_string, retry_count=0) + + def self.perform(user_id, encoded_object_xml, person_ids, dispatcher_class_as_string) user = User.find(user_id) people = Person.where(:id => person_ids) @@ -22,17 +36,17 @@ module Jobs hydra.enqueue_batch hydra.run - unless hydra.failed_people.empty? - if retry_count < MAX_RETRIES - Resque.enqueue(Jobs::HttpMulti, user_id, encoded_object_xml, hydra.failed_people, dispatcher_class_as_string, retry_count + 1 ) + @failed_people = hydra.failed_people + + if not @failed_people.empty? + if self.retry_limit_reached? + msg = "event=http_multi_abandon sender_id=#{user_id} failed_recipient_ids='[#{@failed_people.join(', ')}]'" + Rails.logger.info(msg) else - Rails.logger.info("event=http_multi_abandon sender_id=#{user_id} failed_recipient_ids='[#{person_ids.join(', ')}] '") + raise 'retry' end end end + end end - - - - diff --git a/config/initializers/resque.rb b/config/initializers/resque.rb index 79cbd2995..3a7b0d1a5 100644 --- a/config/initializers/resque.rb +++ b/config/initializers/resque.rb @@ -1,4 +1,6 @@ require 'resque' +require 'resque_scheduler' +require 'resque/scheduler' Resque::Plugins::Timeout.timeout = 300 diff --git a/lib/resque_job_logging.rb b/lib/resque_job_logging.rb index 3402f51a1..9ae016327 100644 --- a/lib/resque_job_logging.rb +++ b/lib/resque_job_logging.rb @@ -22,11 +22,14 @@ module ResqueJobLogging backtrace = application_trace(error) log_string << "app_backtrace='#{backtrace.join(";")}' " notify_hoptoad(error, args) if AppConfig[:hoptoad_api_key].present? + + do_log = !self.respond_to?('retry_limit_reached?') || self.retry_limit_reached? else log_string += "status=complete " + do_log = true end - Rails.logger.info(log_string) + Rails.logger.info(log_string) if do_log raise error if error end diff --git a/lib/tasks/resque.rake b/lib/tasks/resque.rake index 8b582cbfe..b76ae920a 100644 --- a/lib/tasks/resque.rake +++ b/lib/tasks/resque.rake @@ -1,8 +1,12 @@ require 'resque/tasks' +require 'resque_scheduler/tasks' task "resque:setup" do require File.join(File.dirname(__FILE__), '..', '..', 'config', 'environment') Rails.logger.info("event=resque_setup rails_env=#{Rails.env}") + + require 'resque_scheduler' + require 'resque/scheduler' end desc "Alias for resque:work (To run workers on Heroku)" diff --git a/spec/models/jobs/http_multi_spec.rb b/spec/models/jobs/http_multi_spec.rb index cf0061c00..07915b4ab 100644 --- a/spec/models/jobs/http_multi_spec.rb +++ b/spec/models/jobs/http_multi_spec.rb @@ -31,28 +31,6 @@ describe Jobs::HttpMulti do Jobs::HttpMulti.perform(bob.id, @post_xml, people_ids, "Postzord::Dispatcher::Private") end - it 'retries' do - person = @people[0] - - @hydra.stub(:post, person.receive_url).and_return(@failed_response) - - Typhoeus::Hydra.stub!(:new).and_return(@hydra) - - Resque.should_receive(:enqueue).with(Jobs::HttpMulti, bob.id, @post_xml, [person.id], anything, 1).once - Jobs::HttpMulti.perform(bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private") - end - - it 'max retries' do - person = @people[0] - - @hydra.stub(:post, person.receive_url).and_return(@failed_response) - - Typhoeus::Hydra.stub!(:new).and_return(@hydra) - - Resque.should_not_receive(:enqueue) - Jobs::HttpMulti.perform(bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private", 3) - end - it 'generates encrypted xml for people' do person = @people[0] @@ -76,7 +54,12 @@ describe Jobs::HttpMulti do Typhoeus::Hydra.stub!(:new).and_return(@hydra) - Jobs::HttpMulti.perform(bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private") + begin + Jobs::HttpMulti.perform(bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private") + rescue RuntimeError => e + e.message == 'retry' + end + person.reload person.url.should == "https://remote.net/" end diff --git a/spec/support/fake_resque.rb b/spec/support/fake_resque.rb index e08eb0a03..1d4e61a6b 100644 --- a/spec/support/fake_resque.rb +++ b/spec/support/fake_resque.rb @@ -1,7 +1,11 @@ module Resque def enqueue(klass, *args) if $process_queue - klass.send(:perform, *args) + begin + klass.send(:perform, *args) + rescue RuntimeError => e + e.message == 'retry' + end else true end