From 51aca4506f65615571c0bd149b5ea19c3bb99530 Mon Sep 17 00:00:00 2001 From: Benjamin Neff Date: Sat, 11 Jun 2016 00:55:54 +0200 Subject: [PATCH] remove HttpMulti and HydraWrapper --- app/workers/http_multi.rb | 43 -------- config/initializers/load_libraries.rb | 1 - lib/hydra_wrapper.rb | 100 ------------------ lib/postzord/dispatcher.rb | 6 -- spec/lib/hydra_wrapper_spec.rb | 77 -------------- spec/lib/postzord/dispatcher_spec.rb | 19 ---- spec/spec_helper.rb | 1 - spec/support/fake_typhoeus.rb | 22 ---- spec/workers/http_multi_spec.rb | 140 -------------------------- 9 files changed, 409 deletions(-) delete mode 100644 app/workers/http_multi.rb delete mode 100644 lib/hydra_wrapper.rb delete mode 100644 spec/lib/hydra_wrapper_spec.rb delete mode 100644 spec/support/fake_typhoeus.rb delete mode 100644 spec/workers/http_multi_spec.rb diff --git a/app/workers/http_multi.rb b/app/workers/http_multi.rb deleted file mode 100644 index dfc1b9591..000000000 --- a/app/workers/http_multi.rb +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (c) 2010-2011, Diaspora Inc. This file is -# licensed under the Affero General Public License version 3 or later. See -# the COPYRIGHT file. - -module Workers - class HttpMulti < Base - sidekiq_options queue: :http - - MAX_RETRIES = 3 - ABANDON_ON_CODES=[:peer_failed_verification, # Certificate does not match URL - :ssl_connect_error, # Problem negotiating ssl version or Cert couldn't be verified (often self-signed) - :ssl_cacert, # Expired SSL cert - ] - def perform(user_id, encoded_object_xml, person_ids, dispatcher_class_as_string, retry_count=0) - user = User.find(user_id) - people = Person.where(:id => person_ids) - - dispatcher = dispatcher_class_as_string.constantize - hydra = HydraWrapper.new(user, people, encoded_object_xml, dispatcher) - - hydra.enqueue_batch - - hydra.keep_for_retry_if do |response| - !ABANDON_ON_CODES.include?(response.return_code) - end - - hydra.run - - - unless hydra.people_to_retry.empty? - if retry_count < MAX_RETRIES - Workers::HttpMulti.perform_in(1.hour, user_id, encoded_object_xml, hydra.people_to_retry, dispatcher_class_as_string, retry_count + 1) - else - logger.info "event=http_multi_abandon sender_id=#{user_id} failed_recipient_ids='[#{person_ids.join(', ')}]'" - end - end - end - end -end - - - - diff --git a/config/initializers/load_libraries.rb b/config/initializers/load_libraries.rb index 164f3b839..ac23495fc 100644 --- a/config/initializers/load_libraries.rb +++ b/config/initializers/load_libraries.rb @@ -14,7 +14,6 @@ require 'diaspora' require 'direction_detector' require 'email_inviter' require 'evil_query' -require 'hydra_wrapper' require 'postzord' require 'publisher' require 'pubsubhubbub' diff --git a/lib/hydra_wrapper.rb b/lib/hydra_wrapper.rb deleted file mode 100644 index d9d692113..000000000 --- a/lib/hydra_wrapper.rb +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright (c) 2010-2011, Diaspora Inc. This file is -# licensed under the Affero General Public License version 3 or later. See -# the COPYRIGHT file. - -class HydraWrapper - include Diaspora::Logging - - OPTS = { - maxredirs: 3, - timeout: 25, - method: :post, - verbose: AppConfig.settings.typhoeus_verbose?, - cainfo: AppConfig.environment.certificate_authorities.get, - headers: { - 'Expect' => '', - 'Transfer-Encoding' => '', - 'User-Agent' => "Diaspora #{AppConfig.version_string}" - } - } - - attr_reader :people_to_retry , :user, :encoded_object_xml - attr_accessor :dispatcher_class, :people - delegate :run, to: :hydra - - def initialize user, people, encoded_object_xml, dispatcher_class - @user = user - @people_to_retry = [] - @people = people - @dispatcher_class = dispatcher_class - @encoded_object_xml = encoded_object_xml - @keep_for_retry_proc = Proc.new do |response| - true - end - end - - # Inserts jobs for all @people - def enqueue_batch - grouped_people.each do |receive_url, people_for_receive_url| - if xml = xml_factory.xml_for(people_for_receive_url.first) - insert_job(receive_url, xml, people_for_receive_url) - end - end - end - - # This method can be used to tell the hydra whether or not to - # retry a request that it made which failed. - # @yieldparam response [Typhoeus::Response] The response object for the failed request. - # @yieldreturn [Boolean] Whether the request whose response was passed to the block should be retried. - def keep_for_retry_if &block - @keep_for_retry_proc = block - end - - private - - def hydra - @hydra ||= Typhoeus::Hydra.new(max_concurrency: AppConfig.settings.typhoeus_concurrency.to_i) - end - - # @return [Salmon] - def xml_factory - @xml_factory ||= @dispatcher_class.salmon @user, Base64.decode64(@encoded_object_xml) - end - - # Group people on their receiving_urls - # @return [Hash] People grouped by receive_url ([String] => [Array]) - def grouped_people - @people.group_by { |person| - @dispatcher_class.receive_url_for person - } - end - - # Prepares and inserts job into the hydra queue - # @param url [String] - # @param xml [String] - # @params people [Array] - def insert_job url, xml, people - request = Typhoeus::Request.new url, OPTS.merge(body: {xml: CGI.escape(xml)}) - prepare_request request, people - hydra.queue request - end - - # @param request [Typhoeus::Request] - # @param person [Person] - def prepare_request request, people_for_receive_url - request.on_complete do |response| - # Save the reference to the pod to the database if not already present - Pod.find_or_create_by(url: response.effective_url) - - unless response.success? - logger.warn "event=http_multi_fail sender_id=#{@user.id} url=#{response.effective_url} " \ - "return_code=#{response.return_code} response_code=#{response.response_code}" - - if @keep_for_retry_proc.call(response) - @people_to_retry += people_for_receive_url.map(&:id) - end - - end - end - end -end diff --git a/lib/postzord/dispatcher.rb b/lib/postzord/dispatcher.rb index a287b8608..8eb69346f 100644 --- a/lib/postzord/dispatcher.rb +++ b/lib/postzord/dispatcher.rb @@ -94,12 +94,6 @@ class Postzord::Dispatcher # @param remote_people [Array] Recipients of the post on other pods # @return [void] def queue_remote_delivery_job(remote_people) - Workers::HttpMulti.perform_async( - @sender.id, - Base64.strict_encode64(@object.to_diaspora_xml), - remote_people.map{|p| p.id}, - self.class.to_s - ) end # @param people [Array] Recipients of the post diff --git a/spec/lib/hydra_wrapper_spec.rb b/spec/lib/hydra_wrapper_spec.rb deleted file mode 100644 index 10eb94f54..000000000 --- a/spec/lib/hydra_wrapper_spec.rb +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) 2010-2011, Diaspora Inc. This file is -# licensed under the Affero General Public License version 3 or later. See -# the COPYRIGHT file. - -require 'spec_helper' - -describe HydraWrapper do - before do - @people = ["person", "person2", "person3"] - @wrapper = HydraWrapper.new double, @people, "", double - end - - describe 'initialize' do - it 'it sets the proper instance variables' do - user = "user" - encoded_object_xml = "encoded xml" - dispatcher_class = "Postzord::Dispatcher::Private" - - wrapper = HydraWrapper.new user, @people, encoded_object_xml, dispatcher_class - expect(wrapper.user).to eq(user) - expect(wrapper.people).to eq(@people) - expect(wrapper.encoded_object_xml).to eq(encoded_object_xml) - end - end - - describe '#run' do - it 'delegates #run to the @hydra' do - hydra = double.as_null_object - @wrapper.instance_variable_set :@hydra, hydra - expect(hydra).to receive :run - @wrapper.run - end - end - - describe '#xml_factory' do - it 'calls the salmon method on the dispatcher class (and memoizes)' do - allow(Base64).to receive(:decode64).and_return "#{@wrapper.encoded_object_xml} encoded" - decoded = Base64.decode64 @wrapper.encoded_object_xml - expect(@wrapper.dispatcher_class).to receive(:salmon).with(@wrapper.user, decoded).once.and_return true - @wrapper.send :xml_factory - @wrapper.send :xml_factory - end - end - - describe '#grouped_people' do - it 'groups people given their receive_urls' do - expect(@wrapper.dispatcher_class).to receive(:receive_url_for).and_return "foo.com", "bar.com", "bar.com" - - expect(@wrapper.send(:grouped_people)).to eq({"foo.com" => [@people[0]], "bar.com" => @people[1,2]}) - end - end - - describe '#enqueue_batch' do - it 'calls #grouped_people' do - expect(@wrapper).to receive(:grouped_people).and_return [] - @wrapper.enqueue_batch - end - - it 'inserts a job for every group of people' do - allow(Base64).to receive(:decode64) - @wrapper.dispatcher_class = double salmon: double(xml_for: "") - allow(@wrapper).to receive(:grouped_people).and_return('https://foo.com' => @wrapper.people) - expect(@wrapper.people).to receive(:first).once - expect(@wrapper).to receive(:insert_job).with('https://foo.com', "", @wrapper.people).once - @wrapper.enqueue_batch - end - - it 'does not insert a job for a person whos xml returns false' do - allow(Base64).to receive(:decode64) - allow(@wrapper).to receive(:grouped_people).and_return('https://foo.com' => [double]) - @wrapper.dispatcher_class = double salmon: double(xml_for: false) - expect(@wrapper).not_to receive :insert_job - @wrapper.enqueue_batch - end - - end -end diff --git a/spec/lib/postzord/dispatcher_spec.rb b/spec/lib/postzord/dispatcher_spec.rb index 78ed2b126..89bc5e3cd 100644 --- a/spec/lib/postzord/dispatcher_spec.rb +++ b/spec/lib/postzord/dispatcher_spec.rb @@ -184,24 +184,6 @@ describe Postzord::Dispatcher do end end - describe '#deliver_to_remote' do - before do - @remote_people = [] - @remote_people << alice.person - @mailman = Postzord::Dispatcher.build(alice, @sm) - @hydra = double() - allow(Typhoeus::Hydra).to receive(:new).and_return(@hydra) - end - - it 'should queue an HttpMultiJob for the remote people' do - allow_any_instance_of(Postzord::Dispatcher::Public).to receive(:deliver_to_remote).and_call_original - expect(Workers::HttpMulti).to receive(:perform_async).with(alice.id, anything, @remote_people.map{|p| p.id}, anything).once - @mailman.send(:deliver_to_remote, @remote_people) - - allow(Postzord::Dispatcher::Public).to receive(:deliver_to_remote) - end - end - describe '#deliver_to_local' do before do @mailman = Postzord::Dispatcher.build(alice, @sm) @@ -284,7 +266,6 @@ describe Postzord::Dispatcher do mailman = Postzord::Dispatcher.build(alice, FactoryGirl.create(:status_message), :url => "http://joindiaspora.com/p/123", :services => [@s1]) allow(Workers::PublishToHub).to receive(:perform_async).with(anything) - allow(Workers::HttpMulti).to receive(:perform_async).with(anything, anything, anything) expect(Workers::PostToService).to receive(:perform_async).with(@s1.id, anything, anything) mailman.post end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 261350a60..e151f20f0 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -105,7 +105,6 @@ RSpec.configure do |config| config.before(:each) do I18n.locale = :en stub_request(:post, "https://pubsubhubbub.appspot.com/") - disable_typhoeus $process_queue = false allow(Workers::SendPublic).to receive(:perform_async) allow(Workers::SendPrivate).to receive(:perform_async) diff --git a/spec/support/fake_typhoeus.rb b/spec/support/fake_typhoeus.rb deleted file mode 100644 index fdbacaba4..000000000 --- a/spec/support/fake_typhoeus.rb +++ /dev/null @@ -1,22 +0,0 @@ -class FakeHydra - def queue(*args); end - def run; end -end - -class FakeHydraRequest - def initialize(*args); end - def on_complete; end -end - -def disable_typhoeus - silence_warnings do - Workers::HttpMulti.const_set('Hydra', FakeHydra) - Workers::HttpMulti.const_set('Request', FakeHydraRequest) - end -end -def enable_typhoeus - silence_warnings do - Workers::HttpMulti.const_set('Hydra', Typhoeus::Hydra) - Workers::HttpMulti.const_set('Request', Typhoeus::Request) - end -end diff --git a/spec/workers/http_multi_spec.rb b/spec/workers/http_multi_spec.rb deleted file mode 100644 index 5e4202e47..000000000 --- a/spec/workers/http_multi_spec.rb +++ /dev/null @@ -1,140 +0,0 @@ -require 'spec_helper' - -describe Workers::HttpMulti do - before :all do - WebMock.disable_net_connect! allow_localhost: true - WebMock::HttpLibAdapters::TyphoeusAdapter.disable! - enable_typhoeus - end - after :all do - disable_typhoeus - WebMock.disable_net_connect! - end - - before do - @people = [FactoryGirl.create(:person), FactoryGirl.create(:person)] - @post_xml = Base64.encode64 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAH" - - @hydra = Typhoeus::Hydra.new - allow(Typhoeus::Hydra).to receive(:new).and_return(@hydra) - @salmon = Salmon::EncryptedSlap.create_by_user_and_activity bob, Base64.decode64(@post_xml) - allow(Salmon::EncryptedSlap).to receive(:create_by_user_and_activity).and_return @salmon - @body = "encrypted things" - allow(@salmon).to receive(:xml_for).and_return @body - - @response = Typhoeus::Response.new( - code: 200, - body: "", - time: 0.2, - effective_url: 'http://foobar.com', - return_code: :ok - ) - @failed_response = Typhoeus::Response.new( - code: 504, - body: "", - time: 0.2, - effective_url: 'http://foobar.com', - return_code: :ok - ) - @ssl_error_response = Typhoeus::Response.new( - code: 0, - body: "", - time: 0.2, - effective_url: 'http://foobar.com', - return_code: :ssl_connect_error - ) - @unable_to_resolve_response = Typhoeus::Response.new( - code: 0, - body: "", - time: 0.2, - effective_url: 'http://foobar.com', - return_code: :couldnt_resolve_host - ) - end - - it 'POSTs to more than one person' do - @people.each do |person| - Typhoeus.stub(person.receive_url).and_return @response - end - - expect(@hydra).to receive(:queue).twice - expect(@hydra).to receive(:run).once - - Workers::HttpMulti.new.perform bob.id, @post_xml, @people.map(&:id), "Postzord::Dispatcher::Private" - end - - it 'retries' do - person = @people.first - - Typhoeus.stub(person.receive_url).and_return @failed_response - - expect(Workers::HttpMulti).to receive(:perform_in).with(1.hour, bob.id, @post_xml, [person.id], anything, 1).once - Workers::HttpMulti.new.perform bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private" - end - - it 'retries if it could not resolve the server' do - person = @people.first - - Typhoeus.stub(person.receive_url).and_return @unable_to_resolve_response - - expect(Workers::HttpMulti).to receive(:perform_in).with(1.hour, bob.id, @post_xml, [person.id], anything, 1).once - Workers::HttpMulti.new.perform bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private" - end - - it 'does not retry on an SSL error' do - person = @people.first - - Typhoeus.stub(person.receive_url).and_return @ssl_error_response - - expect(Workers::HttpMulti).not_to receive(:perform_in) - Workers::HttpMulti.new.perform bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private" - end - - it 'max retries' do - person = @people.first - - Typhoeus.stub(person.receive_url).and_return @failed_response - - expect(Workers::HttpMulti).not_to receive :perform_in - Workers::HttpMulti.new.perform bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private", 3 - end - - it 'generates encrypted xml for people' do - person = @people.first - - Typhoeus.stub(person.receive_url).and_return @response - expect(@salmon).to receive(:xml_for).and_return @body - - Workers::HttpMulti.new.perform bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private" - end - - it "updates http users who have moved to https" do - person = @people.first - - response = Typhoeus::Response.new( - code: 301, - effective_url: "https://example.net", - response_headers: "Location: #{person.receive_url.sub('http://', 'https://')}", - body: "", - time: 0.2 - ) - Typhoeus.stub(person.receive_url).and_return response - - Workers::HttpMulti.new.perform bob.id, @post_xml, [person.id], "Postzord::Dispatcher::Private" - expect(Person.find(person.id).url).to eq("https://example.net/") - end - - it 'only sends to users with valid RSA keys' do - person = @people.first - person.serialized_public_key = "-----BEGIN RSA PUBLIC KEY-----\nPsych!\n-----END RSA PUBLIC KEY-----" - person.save - - allow(@salmon).to receive(:xml_for).and_call_original - - Typhoeus.stub(person.receive_url).and_return @response - Typhoeus.stub(@people[1].receive_url).and_return @response - - expect(@hydra).to receive(:queue).once - Workers::HttpMulti.new.perform bob.id, @post_xml, @people.map(&:id), "Postzord::Dispatcher::Private" - end -end