Retry federation if remote pod is down

Gemfile

@@ -71,6 +71,8 @@ gem 'ruby-oembed'
 gem 'resque', '1.10.0'
 gem 'resque-ensure-connected'
 gem 'resque-timeout', '1.0.0'
+gem 'resque-scheduler'
+gem 'resque-retry'
 gem 'SystemTimer', '1.2.1', :platforms => :ruby_18
 
 # reporting

Gemfile.lock

@@ -381,6 +381,13 @@ GEM
     resque-ensure-connected (0.1.0)
       activerecord (>= 2.3.5)
       resque (~> 1.10.0)
+    resque-retry (0.1.0)
+      resque (>= 1.8.0)
+      resque-scheduler (>= 1.8.0)
+    resque-scheduler (1.9.9)
+      redis (>= 2.0.1)
+      resque (>= 1.8.0)
+      rufus-scheduler
     resque-timeout (1.0.0)
       resque (~> 1.0)
     rest-client (1.6.1)
@@ -422,6 +429,8 @@ GEM
       archive-tar-minitar (>= 0.5.2)
     rubyntlm (0.1.1)
     rubyzip (0.9.4)
+    rufus-scheduler (2.0.11)
+      tzinfo (>= 0.3.23)
     sass (3.1.7)
     selenium-webdriver (2.7.0)
       childprocess (>= 0.2.1)
@@ -533,6 +542,8 @@ DEPENDENCIES
   redcarpet (= 2.0.0b5)
   resque (= 1.10.0)
   resque-ensure-connected
+  resque-retry
+  resque-scheduler
   resque-timeout (= 1.0.0)
   rest-client (= 1.6.1)
   roxml!

app/models/jobs/http_multi.rb

@@ -3,16 +3,30 @@
 #   the COPYRIGHT file.
 
 require 'uri'
+require 'resque-retry'
 require File.join(Rails.root, 'lib/hydra_wrapper')
 
 module Jobs
   class HttpMulti < Base
+    extend Resque::Plugins::ExponentialBackoff
 
     @queue = :http
+    @backoff_strategy = [10.seconds,
+                          1.minute,
+                         10.minutes,
+                          1.hour,
+                          3.hours,
+                          6.hours,
+                         12.hours,
+                          1.day,
+                          2.days]
 
-    MAX_RETRIES = 3
+    def self.args_for_retry(user_id, encoded_object_xml, person_ids, dispatcher_class_as_string)
+      [user_id, encoded_object_xml, @failed_people, dispatcher_class_as_string]
+    end
 
-    def self.perform(user_id, encoded_object_xml, person_ids, dispatcher_class_as_string, retry_count=0)
+
+    def self.perform(user_id, encoded_object_xml, person_ids, dispatcher_class_as_string)
       user = User.find(user_id)
       people = Person.where(:id => person_ids)
 
@@ -22,17 +36,17 @@ module Jobs
       hydra.enqueue_batch
       hydra.run
 
-      unless hydra.failed_people.empty?
-        if retry_count < MAX_RETRIES
-          Resque.enqueue(Jobs::HttpMulti, user_id, encoded_object_xml, hydra.failed_people, dispatcher_class_as_string, retry_count + 1 )
+      @failed_people = hydra.failed_people      
+
+      if not @failed_people.empty?
+        if self.retry_limit_reached?
+          msg = "event=http_multi_abandon sender_id=#{user_id} failed_recipient_ids='[#{@failed_people.join(', ')}]'"
+          Rails.logger.info(msg)
         else
-          Rails.logger.info("event=http_multi_abandon sender_id=#{user_id} failed_recipient_ids='[#{person_ids.join(', ')}] '")
+          raise 'retry'
         end
       end
     end
+
   end
 end
-
-
-
-

config/initializers/resque.rb

@@ -1,4 +1,6 @@
 require 'resque'
+require 'resque_scheduler'
+require 'resque/scheduler'
 
 Resque::Plugins::Timeout.timeout = 300
 

lib/resque_job_logging.rb

@@ -22,11 +22,14 @@ module ResqueJobLogging
       backtrace = application_trace(error)
       log_string << "app_backtrace='#{backtrace.join(";")}' "
       notify_hoptoad(error, args) if AppConfig[:hoptoad_api_key].present?
+
+      do_log = !self.respond_to?('retry_limit_reached?') || self.retry_limit_reached?
     else
       log_string += "status=complete "
+      do_log = true
     end
 
-    Rails.logger.info(log_string)
+    Rails.logger.info(log_string) if do_log
     raise error if error
   end
 

lib/tasks/resque.rake

@@ -1,8 +1,12 @@
 require 'resque/tasks'
+require 'resque_scheduler/tasks'
 
 task "resque:setup" do
   require File.join(File.dirname(__FILE__), '..', '..', 'config', 'environment')
   Rails.logger.info("event=resque_setup rails_env=#{Rails.env}")
+
+  require 'resque_scheduler'
+  require 'resque/scheduler'
 end
 
 desc "Alias for resque:work (To run workers on Heroku)"

spec/models/jobs/http_multi_spec.rb

@@ -31,28 +31,6 @@ describe Jobs::HttpMulti do
     Jobs::HttpMulti.perform(bob.id, @post_xml, people_ids, "Postzord::Dispatcher::Private")
   end
 
-  it 'retries' do
-    person = @people[0]
-
-    @hydra.stub(:post, person.receive_url).and_return(@failed_response)
-
-    Typhoeus::Hydra.stub!(:new).and_return(@hydra)
-
-    Resque.should_receive(:enqueue).with(Jobs::HttpMulti, bob.id, @post_xml, [person.id], anything, 1).once
-    Jobs::HttpMulti.perform(bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private")
-  end
-
-  it 'max retries' do
-    person = @people[0]
-
-    @hydra.stub(:post, person.receive_url).and_return(@failed_response)
-
-    Typhoeus::Hydra.stub!(:new).and_return(@hydra)
-
-    Resque.should_not_receive(:enqueue)
-    Jobs::HttpMulti.perform(bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private", 3)
-  end
-
   it 'generates encrypted xml for people' do
     person = @people[0]
 
@@ -76,7 +54,12 @@ describe Jobs::HttpMulti do
 
     Typhoeus::Hydra.stub!(:new).and_return(@hydra)
 
-    Jobs::HttpMulti.perform(bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private")
+    begin
+      Jobs::HttpMulti.perform(bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private")
+    rescue RuntimeError => e
+      e.message == 'retry'
+    end
+
     person.reload
     person.url.should == "https://remote.net/"
   end

spec/support/fake_resque.rb

@@ -1,7 +1,11 @@
 module Resque
   def enqueue(klass, *args)
     if $process_queue
-      klass.send(:perform, *args)
+      begin
+        klass.send(:perform, *args)
+      rescue RuntimeError => e
+        e.message == 'retry'
+      end
     else
       true
     end