diff --git a/Changelog.md b/Changelog.md index 157bee633..0757be3aa 100644 --- a/Changelog.md +++ b/Changelog.md @@ -12,6 +12,7 @@ * Use Bootstrap 3 progress-bar for polls [#7600](https://github.com/diaspora/diaspora/pull/7600) * Enable frozen string literals [#7595](https://github.com/diaspora/diaspora/pull/7595) * Remove `rails_admin_histories` table [#7597](https://github.com/diaspora/diaspora/pull/7597) +* Optimize memory usage on profile export [#7627](https://github.com/diaspora/diaspora/pull/7627) ## Bug fixes * Fix displaying polls with long answers [#7579](https://github.com/diaspora/diaspora/pull/7579) diff --git a/app/serializers/export/others_data_serializer.rb b/app/serializers/export/others_data_serializer.rb index fe53579e1..eff06a806 100644 --- a/app/serializers/export/others_data_serializer.rb +++ b/app/serializers/export/others_data_serializer.rb @@ -4,33 +4,32 @@ module Export class OthersDataSerializer < ActiveModel::Serializer # Relayables of other people in the archive: comments, likes, participations, poll participations where author is # the archive owner - has_many :relayables, each_serializer: FederationEntitySerializer + has_many :relayables, serializer: FlatMapArraySerializer, each_serializer: FederationEntitySerializer - # Parent posts of user's own relayables. We have to save metadata to use - # it in case when posts temporary unavailable on the target pod. - has_many :posts, each_serializer: FederationEntitySerializer - - # Authors of posts where we participated and authors are not in contacts - has_many :non_contact_authors, each_serializer: PersonMetadataSerializer + def initialize(user_id) + @user_id = user_id + super(object) + end private + def object + User.find(@user_id) + end + def relayables %i[comments likes poll_participations].map {|relayable| - others_relayables.send(relayable) - }.sum + others_relayables.send(relayable).find_each(batch_size: 20) + } end def others_relayables @others_relayables ||= Diaspora::Exporter::OthersRelayables.new(object.person_id) end - def posts - @posts ||= Diaspora::Exporter::PostsWithActivity.new(object).query - end - - def non_contact_authors - Diaspora::Exporter::NonContactAuthors.new(posts, object).query + # Avoid calling pointless #embedded_in_root_associations method + def serializable_data + {} end end end diff --git a/app/serializers/export/own_post_serializer.rb b/app/serializers/export/own_post_serializer.rb index bea797aaa..2813253e5 100644 --- a/app/serializers/export/own_post_serializer.rb +++ b/app/serializers/export/own_post_serializer.rb @@ -29,7 +29,7 @@ module Export end def excluded_subscription_key - entity.public ? :subscribed_users_ids : :subscribed_pods_uris + object.public? ? :subscribed_users_ids : :subscribed_pods_uris end end end diff --git a/app/serializers/export/person_metadata_serializer.rb b/app/serializers/export/person_metadata_serializer.rb deleted file mode 100644 index b309a7d12..000000000 --- a/app/serializers/export/person_metadata_serializer.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module Export - class PersonMetadataSerializer < ActiveModel::Serializer - attributes :guid, - :account_id, - :public_key - - private - - def account_id - object.diaspora_handle - end - - def public_key - object.serialized_public_key - end - end -end diff --git a/app/serializers/export/user_serializer.rb b/app/serializers/export/user_serializer.rb index c571bdf47..f1c4db5a3 100644 --- a/app/serializers/export/user_serializer.rb +++ b/app/serializers/export/user_serializer.rb @@ -18,12 +18,31 @@ module Export has_many :followed_tags has_many :post_subscriptions - has_many :relayables, each_serializer: Export::OwnRelayablesSerializer + has_many :relayables, serializer: FlatMapArraySerializer, each_serializer: Export::OwnRelayablesSerializer + + def initialize(user_id, options={}) + @user_id = user_id + super(object, options) + end private + def object + User.find(@user_id) + end + + def posts + object.posts.find_each(batch_size: 20) + end + + def contacts + object.contacts.find_each(batch_size: 100) + end + def relayables - [*comments, *likes, *poll_participations] + [comments, likes, poll_participations].map {|relayable| + relayable.find_each(batch_size: 20) + } end %i[comments likes poll_participations].each {|collection| @@ -47,5 +66,10 @@ module Export def post_subscriptions Post.subscribed_by(object).pluck(:guid) end + + # Avoid calling pointless #embedded_in_root_associations method + def serializable_data + {} + end end end diff --git a/app/serializers/federation_entity_serializer.rb b/app/serializers/federation_entity_serializer.rb index 5bc620171..22fa74afc 100644 --- a/app/serializers/federation_entity_serializer.rb +++ b/app/serializers/federation_entity_serializer.rb @@ -13,6 +13,6 @@ class FederationEntitySerializer < ActiveModel::Serializer end def entity - @entity ||= Diaspora::Federation::Entities.build(object) + Diaspora::Federation::Entities.build(object) end end diff --git a/app/serializers/flat_map_array_serializer.rb b/app/serializers/flat_map_array_serializer.rb new file mode 100644 index 000000000..18f45cd8f --- /dev/null +++ b/app/serializers/flat_map_array_serializer.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class FlatMapArraySerializer < ActiveModel::ArraySerializer + def serializable_object(options={}) + @object.flat_map do |subarray| + subarray.map do |item| + serializer_for(item).serializable_object_with_notification(options) + end + end + end +end diff --git a/lib/diaspora/exporter.rb b/lib/diaspora/exporter.rb index cbdecc10d..837261b76 100644 --- a/lib/diaspora/exporter.rb +++ b/lib/diaspora/exporter.rb @@ -20,8 +20,8 @@ module Diaspora def full_archive {version: SERIALIZED_VERSION} - .merge(Export::UserSerializer.new(@user).as_json) - .merge(Export::OthersDataSerializer.new(@user).as_json) + .merge(Export::UserSerializer.new(@user.id).as_json) + .merge(Export::OthersDataSerializer.new(@user.id).as_json) end end end diff --git a/lib/diaspora/exporter/non_contact_authors.rb b/lib/diaspora/exporter/non_contact_authors.rb deleted file mode 100644 index 1c89f8d8d..000000000 --- a/lib/diaspora/exporter/non_contact_authors.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Diaspora - class Exporter - # This class is capable of quering a list of people from authors of given posts that are non-contacts of a given - # user. - class NonContactAuthors - # @param posts [Post::ActiveRecord_Relation] posts that we fetch authors from to make authors list - # @param user [User] a user we fetch a contact list from - def initialize(posts, user) - @posts = posts - @user = user - end - - # Create a request of non-contact authors of the posts for the user - # @return [Post::ActiveRecord_Relation] - def query - Person.where(id: non_contact_authors_ids) - end - - private - - def non_contact_authors_ids - posts_authors_ids - contacts_ids - end - - def posts_authors_ids - posts.pluck(:author_id).uniq - end - - def contacts_ids - user.contacts.pluck(:person_id) - end - - attr_reader :posts, :user - end - end -end diff --git a/lib/diaspora/exporter/posts_with_activity.rb b/lib/diaspora/exporter/posts_with_activity.rb deleted file mode 100644 index 9ebafa759..000000000 --- a/lib/diaspora/exporter/posts_with_activity.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -module Diaspora - class Exporter - # This class allows to query posts where a person made any activity (submitted comments, - # likes, participations or poll participations). - class PostsWithActivity - # @param user [User] user who the activity belongs to (the one who liked, commented posts, etc) - def initialize(user) - @user = user - end - - # Create a request of posts with activity - # @return [Post::ActiveRecord_Relation] - def query - Post.from("(#{sql_union_all_activities}) AS posts") - end - - private - - attr_reader :user - - def person - user.person - end - - def sql_union_all_activities - all_activities.map(&:to_sql).join(" UNION ") - end - - def all_activities - [comments_activity, likes_activity, subscriptions, polls_activity, reshares_activity] - end - - def likes_activity - other_people_posts.liked_by(person) - end - - def comments_activity - other_people_posts.commented_by(person) - end - - def subscriptions - other_people_posts.subscribed_by(user) - end - - def reshares_activity - other_people_posts.reshared_by(person) - end - - def polls_activity - StatusMessage.where.not(author_id: person.id).joins(:poll_participations) - .where(poll_participations: {author_id: person.id}) - end - - def other_people_posts - Post.where.not(author_id: person.id) - end - end - end -end diff --git a/spec/integration/exporter_spec.rb b/spec/integration/exporter_spec.rb index 930a8a57e..00110dafa 100644 --- a/spec/integration/exporter_spec.rb +++ b/spec/integration/exporter_spec.rb @@ -201,7 +201,7 @@ describe Diaspora::Exporter do expect(json).to include_json(user: {posts: [serialized]}) end - it "contains a reshare and its root" do + it "contains a reshare" do reshare = FactoryGirl.create(:reshare, author: user.person) serialized_reshare = { "subscribed_pods_uris": [reshare.root.author.pod.url_to(""), AppConfig.pod_uri.to_s], @@ -216,21 +216,8 @@ describe Diaspora::Exporter do } } - status_message = reshare.root - serialized_parent = { - "entity_type": "status_message", - "entity_data": { - "author": status_message.diaspora_handle, - "guid": status_message.guid, - "created_at": status_message.created_at.iso8601, - "text": status_message.text, - "public": true - } - } - expect(json).to include_json( - user: {posts: [serialized_reshare]}, - others_data: {posts: [serialized_parent]} + user: {posts: [serialized_reshare]} ) end @@ -244,7 +231,7 @@ describe Diaspora::Exporter do expect(json).to include_json(user: {post_subscriptions: [subscription.target.guid]}) end - it "contains a comment and the commented post" do + it "contains a comment" do comment = FactoryGirl.create(:comment, author: user.person) serialized_comment = { "entity_type": "comment", @@ -258,25 +245,12 @@ describe Diaspora::Exporter do "property_order": %w[author guid parent_guid text created_at] } - status_message = comment.parent - serialized_post = { - "entity_type": "status_message", - "entity_data": { - "author": status_message.diaspora_handle, - "guid": status_message.guid, - "created_at": status_message.created_at.iso8601, - "text": status_message.text, - "public": false - } - } - expect(json).to include_json( - user: {relayables: [serialized_comment]}, - others_data: {posts: [serialized_post]} + user: {relayables: [serialized_comment]} ) end - it "contains a like and the liked post" do + it "contains a like" do like = FactoryGirl.create(:like, author: user.person) serialized_like = { "entity_type": "like", @@ -290,25 +264,12 @@ describe Diaspora::Exporter do "property_order": %w[author guid parent_guid parent_type positive] } - status_message = like.target - serialized_post = { - "entity_type": "status_message", - "entity_data": { - "author": status_message.diaspora_handle, - "guid": status_message.guid, - "created_at": status_message.created_at.iso8601, - "text": status_message.text, - "public": false - } - } - expect(json).to include_json( - user: {relayables: [serialized_like]}, - others_data: {posts: [serialized_post]} + user: {relayables: [serialized_like]} ) end - it "contains a poll participation and post with this poll" do + it "contains a poll participation" do poll_participation = FactoryGirl.create(:poll_participation, author: user.person) serialized_participation = { "entity_type": "poll_participation", @@ -321,38 +282,8 @@ describe Diaspora::Exporter do "property_order": %w[author guid parent_guid poll_answer_guid] } - poll = poll_participation.poll - status_message = poll_participation.status_message - serialized_post = { - "entity_type": "status_message", - "entity_data": { - "author": status_message.diaspora_handle, - "guid": status_message.guid, - "created_at": status_message.created_at.iso8601, - "text": status_message.text, - "poll": { - "entity_type": "poll", - "entity_data": { - "guid": poll.guid, - "question": poll.question, - "poll_answers": poll.poll_answers.map {|answer| - { - "entity_type": "poll_answer", - "entity_data": { - "guid": answer.guid, - "answer": answer.answer - } - } - } - } - }, - "public": false - } - } - expect(json).to include_json( - user: {relayables: [serialized_participation]}, - others_data: {posts: [serialized_post]} + user: {relayables: [serialized_participation]} ) end @@ -409,23 +340,6 @@ describe Diaspora::Exporter do expect(json).to include_json(others_data: {relayables: [serialized]}) end - it "contains metadata of a non-contact author of a post where we commented" do - comment = FactoryGirl.create(:comment, author: user.person) - - author = comment.parent.author - expect(json).to include_json( - others_data: { - non_contact_authors: [ - { - "guid": author.guid, - "account_id": author.diaspora_handle, - "public_key": author.serialized_public_key - } - ] - } - ) - end - def transform_value(value) return value.iso8601 if value.is_a? Date value diff --git a/spec/lib/diaspora/exporter/non_contact_authors_spec.rb b/spec/lib/diaspora/exporter/non_contact_authors_spec.rb deleted file mode 100644 index 766fce62a..000000000 --- a/spec/lib/diaspora/exporter/non_contact_authors_spec.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -describe Diaspora::Exporter::NonContactAuthors do - describe "#query" do - let(:user) { FactoryGirl.create(:user_with_aspect) } - let(:post) { FactoryGirl.create(:status_message) } - let(:instance) { - Diaspora::Exporter::NonContactAuthors.new(Post.where(id: post.id), user) - } - - context "without contact relationship" do - it "includes post author to the result set" do - expect(instance.query).to eq([post.author]) - end - end - - context "with contact relationship" do - before do - user.share_with(post.author, user.aspects.first) - end - - it "doesn't include post author to the result set" do - expect(instance.query).to be_empty - end - end - end -end diff --git a/spec/lib/diaspora/exporter/posts_with_activity_spec.rb b/spec/lib/diaspora/exporter/posts_with_activity_spec.rb deleted file mode 100644 index f3c2e83ce..000000000 --- a/spec/lib/diaspora/exporter/posts_with_activity_spec.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -describe Diaspora::Exporter::PostsWithActivity do - let(:user) { FactoryGirl.create(:user) } - let(:instance) { Diaspora::Exporter::PostsWithActivity.new(user) } - - describe "#query" do - let(:activity) { - [ - user.person.likes.first.target, - user.person.comments.first.parent, - user.person.poll_participations.first.parent.status_message, - user.person.participations.first.target, - user.person.posts.reshares.first.root - ] - } - - before do - DataGenerator.create(user, %i[activity participation]) - end - - it "returns all posts with person's activity" do - expect(instance.query).to match_array(activity) - end - end -end diff --git a/spec/lib/diaspora/exporter_spec.rb b/spec/lib/diaspora/exporter_spec.rb index 63c4c9c37..6a9b1056f 100644 --- a/spec/lib/diaspora/exporter_spec.rb +++ b/spec/lib/diaspora/exporter_spec.rb @@ -6,7 +6,7 @@ describe Diaspora::Exporter do expect_any_instance_of(Export::UserSerializer).to receive(:as_json).and_return(user: "user_data") expect_any_instance_of(Export::OthersDataSerializer).to receive(:as_json).and_return(others_date: "others_data") - json = Diaspora::Exporter.new(nil).execute + json = Diaspora::Exporter.new(FactoryGirl.create(:user)).execute expect(json).to include_json( version: "2.0", user: "user_data", diff --git a/spec/serializers/export/others_data_serializer_spec.rb b/spec/serializers/export/others_data_serializer_spec.rb index 3c06563aa..2c40adfb8 100644 --- a/spec/serializers/export/others_data_serializer_spec.rb +++ b/spec/serializers/export/others_data_serializer_spec.rb @@ -2,15 +2,7 @@ describe Export::OthersDataSerializer do let(:user) { FactoryGirl.create(:user) } - let(:serializer) { Export::OthersDataSerializer.new(user) } - let(:others_posts) { - [ - *user.person.likes.map(&:target), - *user.person.comments.map(&:parent), - *user.person.posts.reshares.map(&:root), - *user.person.poll_participations.map(&:status_message) - ] - } + let(:serializer) { Export::OthersDataSerializer.new(user.id) } it "uses FederationEntitySerializer for array serializing relayables" do sm = DataGenerator.new(user).status_message_with_activity @@ -25,21 +17,5 @@ describe Export::OthersDataSerializer do before do DataGenerator.new(user).activity end - - it "uses FederationEntitySerializer for array serializing posts" do - expect(Export::OthersDataSerializer).to serialize_association(:posts) - .with_each_serializer(FederationEntitySerializer) - .with_objects(others_posts) - serializer.associations - end - - it "uses PersonMetadataSerializer for array serializing non_contact_authors" do - non_contact_authors = others_posts.map(&:author) - - expect(Export::OthersDataSerializer).to serialize_association(:non_contact_authors) - .with_each_serializer(Export::PersonMetadataSerializer) - .with_objects(non_contact_authors) - serializer.associations - end end end diff --git a/spec/serializers/export/person_metadata_serializer_spec.rb b/spec/serializers/export/person_metadata_serializer_spec.rb deleted file mode 100644 index 8a278da78..000000000 --- a/spec/serializers/export/person_metadata_serializer_spec.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -describe Export::PersonMetadataSerializer do - let(:person) { FactoryGirl.create(:person) } - let(:serializer) { Export::PersonMetadataSerializer.new(person) } - - it "has person metadata attributes" do - expect(serializer.attributes).to eq( - guid: person.guid, - account_id: person.diaspora_handle, - public_key: person.serialized_public_key - ) - end -end diff --git a/spec/serializers/export/user_serializer_spec.rb b/spec/serializers/export/user_serializer_spec.rb index 0340b696c..a316ef3d9 100644 --- a/spec/serializers/export/user_serializer_spec.rb +++ b/spec/serializers/export/user_serializer_spec.rb @@ -2,7 +2,7 @@ describe Export::UserSerializer do let(:user) { FactoryGirl.create(:user) } - let(:serializer) { Export::UserSerializer.new(user, root: false) } + let(:serializer) { Export::UserSerializer.new(user.id, root: false) } it "has basic user's attributes" do expect(serializer.attributes).to eq( diff --git a/spec/support/serializer_matchers.rb b/spec/support/serializer_matchers.rb index 0a1eff813..7da7b6927 100644 --- a/spec/support/serializer_matchers.rb +++ b/spec/support/serializer_matchers.rb @@ -15,6 +15,7 @@ RSpec::Matchers.define :serialize_association do |association_name| match do |root_serializer_class| association = fetch_association(root_serializer_class, association_name) + @serializer_from_options = association.serializer_from_options execute_receive_matcher_with(association) end @@ -50,7 +51,11 @@ RSpec::Matchers.define :serialize_association do |association_name| def with_object_expectation(object) if association_object.is_a?(Array) - expect(object).to match_array(association_object) + if serializer_class == FlatMapArraySerializer + expect(object.flat_map(&:to_a)).to match_array(association_object) + else + expect(object).to match_array(association_object) + end elsif !association_object.nil? expect(object).to eq(association_object) end @@ -66,6 +71,7 @@ RSpec::Matchers.define :serialize_association do |association_name| def pick_serializer_class return association_serializer_class unless association_serializer_class.nil? + return @serializer_from_options unless @serializer_from_options.nil? return ActiveModel::ArraySerializer unless each_serializer_class.nil? end end