From 93d61c7f2166acf8f8fa25f07123afdc444534a9 Mon Sep 17 00:00:00 2001 From: cmrd Senya <35317-cmrd-senya@users.noreply.gitlab.gnome.org> Date: Sun, 4 Sep 2022 22:52:15 +0300 Subject: [PATCH 1/5] Use federation code for fetching public posts on search --- lib/diaspora/fetcher/public.rb | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/diaspora/fetcher/public.rb b/lib/diaspora/fetcher/public.rb index 03530167a..70a159bf5 100644 --- a/lib/diaspora/fetcher/public.rb +++ b/lib/diaspora/fetcher/public.rb @@ -104,16 +104,13 @@ module Diaspora; module Fetcher; class Public logger.debug "post: #{post.to_s[0..250]}" - entry = StatusMessage.diaspora_initialize( - author: @person, - public: true, - guid: post["guid"], - text: post["text"], - provider_display_name: post["provider_display_name"], - created_at: ActiveSupport::TimeZone.new("UTC").parse(post["created_at"]).to_datetime, + DiasporaFederation::Federation::Fetcher.fetch_public( + @person.diaspora_handle, + :post, + post["guid"] ) - entry.save - + rescue DiasporaFederation::Federation::Fetcher::NotFetchable => e + logger.debug e.message end set_fetch_status Public::Status_Processed end From f3c01d5a46e8a2c96906a0db7b7ab637eadd4242 Mon Sep 17 00:00:00 2001 From: cmrd Senya <35317-cmrd-senya@users.noreply.gitlab.gnome.org> Date: Sun, 4 Sep 2022 23:28:43 +0300 Subject: [PATCH 2/5] post fetch: remove check for type to allow fetching reshares --- lib/diaspora/fetcher/public.rb | 12 +----------- spec/lib/diaspora/fetcher/public_spec.rb | 13 ------------- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/lib/diaspora/fetcher/public.rb b/lib/diaspora/fetcher/public.rb index 70a159bf5..30bdac2f6 100644 --- a/lib/diaspora/fetcher/public.rb +++ b/lib/diaspora/fetcher/public.rb @@ -128,9 +128,8 @@ module Diaspora; module Fetcher; class Public # @see check_existing # @see check_author # @see check_public - # @see check_type def validate post - check_existing(post) && check_author(post) && check_public(post) && check_type(post) + check_existing(post) && check_author(post) && check_public(post) end # hopefully there is no post with the same guid somewhere already... @@ -163,13 +162,4 @@ module Diaspora; module Fetcher; class Public ispublic end - - # see, if the type of the given post is something we can handle - def check_type post - type_ok = (post['post_type'] == "StatusMessage") - - logger.warn "the post (#{post['guid']}) has a type, which cannot be handled (#{post['post_type']})" unless type_ok - - type_ok - end end; end; end diff --git a/spec/lib/diaspora/fetcher/public_spec.rb b/spec/lib/diaspora/fetcher/public_spec.rb index 5809073da..3f914175a 100644 --- a/spec/lib/diaspora/fetcher/public_spec.rb +++ b/spec/lib/diaspora/fetcher/public_spec.rb @@ -208,7 +208,6 @@ describe Diaspora::Fetcher::Public do expect(public_fetcher).to receive(:check_existing).and_return(true) expect(public_fetcher).to receive(:check_author).and_return(true) expect(public_fetcher).to receive(:check_public).and_return(true) - expect(public_fetcher).to receive(:check_type).and_return(true) expect(public_fetcher.instance_eval { validate({}) }).to be true end @@ -256,17 +255,5 @@ describe Diaspora::Fetcher::Public do expect(public_fetcher.instance_eval { check_public post }).to be true end end - - describe "#check_type" do - it "returns false if the type is anything other that 'StatusMessage'" do - post = {"post_type"=>"Reshare"} - expect(public_fetcher.instance_eval { check_type post }).to be false - end - - it "returns true if the type is 'StatusMessage'" do - post = {"post_type"=>"StatusMessage"} - expect(public_fetcher.instance_eval { check_type post }).to be true - end - end end end From a6e7c8e2d5c16c6c600a93aeccb87eca2bb1d3fe Mon Sep 17 00:00:00 2001 From: cmrd Senya <35317-cmrd-senya@users.noreply.gitlab.gnome.org> Date: Mon, 5 Sep 2022 23:19:11 +0300 Subject: [PATCH 3/5] post fetch: update post fetch spec --- spec/lib/diaspora/fetcher/public_spec.rb | 58 +++++++++++++++++++----- 1 file changed, 46 insertions(+), 12 deletions(-) diff --git a/spec/lib/diaspora/fetcher/public_spec.rb b/spec/lib/diaspora/fetcher/public_spec.rb index 3f914175a..8f8f405fb 100644 --- a/spec/lib/diaspora/fetcher/public_spec.rb +++ b/spec/lib/diaspora/fetcher/public_spec.rb @@ -4,13 +4,21 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. +require "integration/federation/federation_helper" + # Tests fetching public posts of a person on a remote server describe Diaspora::Fetcher::Public do + let(:fixture) do + File.read(Rails.root.join("spec/fixtures/public_posts.json")) + end + let(:fixture_data) do + JSON.parse(fixture) + end + before do # the fixture is taken from an actual json request. # it contains 10 StatusMessages and 5 Reshares, all of them public # the guid of the person is "7445f9a0a6c28ebb" - @fixture = File.read(Rails.root.join("spec/fixtures/public_posts.json")) @fetcher = Diaspora::Fetcher::Public.new @person = FactoryBot.create(:person, guid: "7445f9a0a6c28ebb", pod: Pod.find_or_create_by(url: "https://remote-testpod.net"), @@ -21,7 +29,7 @@ describe Diaspora::Fetcher::Public do "Accept" => "application/json", "Accept-Encoding" => "gzip;q=1.0,deflate;q=0.6,identity;q=0.3", "User-Agent" => "diaspora-fetcher" - }).to_return(body: @fixture) + }).to_return(body: fixture) end describe "#queue_for" do @@ -62,28 +70,54 @@ describe Diaspora::Fetcher::Public do @data } expect(data).not_to be_nil - expect(data.size).to eql JSON.parse(@fixture).size + expect(data.size).to eq(fixture_data.size) end end describe "#process_posts" do before do person = @person - data = JSON.parse(@fixture) + data = fixture_data @fetcher.instance_eval { @person = person @data = data } + + fixture_data.each do |post_data| + post = if post_data["post_type"] == "StatusMessage" + FactoryBot.build( + :status_message, + guid: post_data["guid"], + text: post_data["text"], + created_at: post_data["created_at"], + public: true, + author: eve.person + ) + else + reshare = FactoryBot.build( + :reshare, + guid: post_data["guid"], + created_at: post_data["created_at"], + public: true, + author: eve.person + ) + reshare.root.save + reshare + end + payload = generate_payload(Diaspora::Federation::Entities.post(post), eve) + + stub_request(:get, "https://remote-testpod.net/fetch/post/#{post_data['guid']}") + .to_return(status: 200, body: payload) + end end - it "creates 10 new posts in the database" do - before_count = Post.count - @fetcher.instance_eval { - process_posts - } - after_count = Post.count - expect(after_count).to eql(before_count + 10) + it "creates 15 new posts in the database" do + expect { + @fetcher.instance_eval { + process_posts + } + }.to change(Post, :count).by(15) end it "sets the operation status on the person" do @@ -100,7 +134,7 @@ describe Diaspora::Fetcher::Public do before do Timecop.freeze @now = DateTime.now.utc - @data = JSON.parse(@fixture).select {|item| item["post_type"] == "StatusMessage" } + @data = fixture_data.select {|item| item["post_type"] == "StatusMessage" } # save posts to db @fetcher.instance_eval { From 61d7eb100a0a36eb6ae7a0b25ceddc7376c71a56 Mon Sep 17 00:00:00 2001 From: cmrd Senya <35317-cmrd-senya@users.noreply.gitlab.gnome.org> Date: Mon, 5 Sep 2022 23:22:26 +0300 Subject: [PATCH 4/5] post fetch: use warn not debug for exception logging --- lib/diaspora/fetcher/public.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/diaspora/fetcher/public.rb b/lib/diaspora/fetcher/public.rb index 30bdac2f6..6b8555fa1 100644 --- a/lib/diaspora/fetcher/public.rb +++ b/lib/diaspora/fetcher/public.rb @@ -110,7 +110,7 @@ module Diaspora; module Fetcher; class Public post["guid"] ) rescue DiasporaFederation::Federation::Fetcher::NotFetchable => e - logger.debug e.message + logger.warn e.message end set_fetch_status Public::Status_Processed end From e77d785d9c01429ea2a45cb9cd0227196a0e692c Mon Sep 17 00:00:00 2001 From: cmrd Senya <35317-cmrd-senya@users.noreply.gitlab.gnome.org> Date: Tue, 6 Sep 2022 10:22:08 +0300 Subject: [PATCH 5/5] post fetch: update spec to test reshares data too --- spec/lib/diaspora/fetcher/public_spec.rb | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/spec/lib/diaspora/fetcher/public_spec.rb b/spec/lib/diaspora/fetcher/public_spec.rb index 8f8f405fb..f0e1f3738 100644 --- a/spec/lib/diaspora/fetcher/public_spec.rb +++ b/spec/lib/diaspora/fetcher/public_spec.rb @@ -134,7 +134,6 @@ describe Diaspora::Fetcher::Public do before do Timecop.freeze @now = DateTime.now.utc - @data = fixture_data.select {|item| item["post_type"] == "StatusMessage" } # save posts to db @fetcher.instance_eval { @@ -147,26 +146,26 @@ describe Diaspora::Fetcher::Public do end it "applies the date from JSON to the record" do - @data.each do |post| + fixture_data.each do |post| date = ActiveSupport::TimeZone.new("UTC").parse(post["created_at"]).to_i - entry = StatusMessage.find_by(guid: post["guid"]) + entry = Post.find_by(guid: post["guid"]) expect(entry.created_at.to_i).to eql(date) end end - it "copied the text correctly" do - @data.each do |post| + it "copied the text of status messages correctly" do + fixture_data.select {|item| item["post_type"] == "StatusMessage" }.each do |post| entry = StatusMessage.find_by(guid: post["guid"]) expect(entry.text).to eql(post["text"]) end end it "applies now to interacted_at on the record" do - @data.each do |post| + fixture_data.each do |post| date = @now.to_i - entry = StatusMessage.find_by(guid: post["guid"]) + entry = Post.find_by(guid: post["guid"]) expect(entry.interacted_at.to_i).to eql(date) end end