From 1c69dd77529903713bf16f2ad0a41b386aa172da Mon Sep 17 00:00:00 2001 From: James Kiesel Date: Sun, 28 Dec 2014 01:44:53 +1300 Subject: [PATCH] Add contacts/posts, and GZipping JSON exporter output --- Changelog.md | 2 +- app/controllers/users_controller.rb | 14 ++++--- app/mailers/export_mailer.rb | 22 +++++++++++ app/models/user.rb | 24 +++++++++++- app/serializers/export/comment_serializer.rb | 10 +++++ app/serializers/export/post_serializer.rb | 14 +++++++ app/serializers/export/user_serializer.rb | 6 +++ app/uploaders/exported_user.rb | 19 ++++++++++ app/views/users/edit.html.haml | 16 +++++++- app/views/users/edit.mobile.haml | 14 ++++++- app/views/users/export_email.markerb | 1 + app/views/users/export_failure_email.markerb | 1 + app/workers/export_user.rb | 21 +++++++++++ config/locales/diaspora/en.yml | 27 +++++++++++++- config/routes.rb | 3 +- .../20141227120907_add_export_to_user.rb | 7 ++++ db/schema.rb | 5 ++- spec/controllers/users_controller_spec.rb | 19 ++++++++-- spec/mailers/export_spec.rb | 37 +++++++++++++++++++ spec/models/user_spec.rb | 27 ++++++++++++++ spec/workers/export_user_spec.rb | 26 +++++++++++++ 21 files changed, 297 insertions(+), 18 deletions(-) create mode 100644 app/mailers/export_mailer.rb create mode 100644 app/serializers/export/comment_serializer.rb create mode 100644 app/serializers/export/post_serializer.rb create mode 100644 app/uploaders/exported_user.rb create mode 100644 app/views/users/export_email.markerb create mode 100644 app/views/users/export_failure_email.markerb create mode 100644 app/workers/export_user.rb create mode 100644 db/migrate/20141227120907_add_export_to_user.rb create mode 100644 spec/mailers/export_spec.rb create mode 100644 spec/workers/export_user_spec.rb diff --git a/Changelog.md b/Changelog.md index a0d4c81e4..e5fa600d8 100644 --- a/Changelog.md +++ b/Changelog.md @@ -137,12 +137,12 @@ diaspora.yml file**. The existing settings from 0.4.x and before will not work a * Truncate too long OpenGraph descriptions [#5387](https://github.com/diaspora/diaspora/pull/5387) * Make the source code URL configurable [#5410](https://github.com/diaspora/diaspora/pull/5410) * Prefill publisher on the tag pages [#5442](https://github.com/diaspora/diaspora/pull/5442) -* Allows users to export their data in JSON format from their user settings page [#5354](https://github.com/diaspora/diaspora/pull/5354) * Don't include the content of non-public posts into notification mails [#5494](https://github.com/diaspora/diaspora/pull/5494) * Allow to set unhosted button and currency for paypal donation [#5452](https://github.com/diaspora/diaspora/pull/5452) * Add followed tags in the mobile menu [#5468](https://github.com/diaspora/diaspora/pull/5468) * Replace Pagedown with markdown-it [#5526](https://github.com/diaspora/diaspora/pull/5526) * Do not truncate notification emails anymore [#4342](https://github.com/diaspora/diaspora/issues/4342) +* Allows users to export their data in gzipped JSON format from their user settings page [#5499](https://github.com/diaspora/diaspora/pull/5499) # 0.4.1.2 diff --git a/app/controllers/users_controller.rb b/app/controllers/users_controller.rb index 91ac6b3d3..fabb3f3a4 100644 --- a/app/controllers/users_controller.rb +++ b/app/controllers/users_controller.rb @@ -135,12 +135,14 @@ class UsersController < ApplicationController redirect_to stream_path end - def export - if export = Diaspora::Exporter.new(current_user).execute - send_data export, filename: "#{current_user.username}_diaspora_data.json", type: :json - else - head :not_acceptable - end + def export_profile + current_user.queue_export + flash[:notice] = I18n.t('users.edit.export_in_progress') + redirect_to edit_user_path + end + + def download_profile + send_data File.open(current_user.export.path).read, type: :json, filename: current_user.export.filename end def export_photos diff --git a/app/mailers/export_mailer.rb b/app/mailers/export_mailer.rb new file mode 100644 index 000000000..8559fa8b1 --- /dev/null +++ b/app/mailers/export_mailer.rb @@ -0,0 +1,22 @@ +class ExportMailer < ActionMailer::Base + default from: AppConfig.mail.sender_address + + def export_complete_for(user) + @user = user + + mail(to: @user.email, subject: I18n.t('notifier.export_email.subject')) do |format| + format.html { render 'users/export_email' } + format.text { render 'users/export_email' } + end.deliver + end + + def export_failure_for(user) + @user = user + + mail(to: @user.email, subject: I18n.t('notifier.export_failure_email.subject')) do |format| + format.html { render 'users/export_failure_email' } + format.text { render 'users/export_failure_email' } + end.deliver + end + +end diff --git a/app/models/user.rb b/app/models/user.rb index 0a7c2d863..3fdbec790 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -291,6 +291,28 @@ class User < ActiveRecord::Base end end + ######### Data export ################## + mount_uploader :export, ExportedUser + + def queue_export + update exporting: true + Workers::ExportUser.perform_async(id) + end + + def perform_export! + export = Tempfile.new([username, '.json.gz'], encoding: 'ascii-8bit') + export.write(compressed_export) && export.close + if export.present? + update exporting: false, export: export, exported_at: Time.zone.now + else + update exporting: false + end + end + + def compressed_export + ActiveSupport::Gzip.compress Diaspora::Exporter.new(self).execute + end + ######### Mailer ####################### def mail(job, *args) pref = job.to_s.gsub('Workers::Mail::', '').underscore @@ -505,6 +527,6 @@ class User < ActiveRecord::Base "created_at", "updated_at", "locked_at", "serialized_private_key", "getting_started", "disable_mail", "show_community_spotlight_in_stream", - "email", "remove_after"] + "email", "remove_after", "export", "exporting", "exported_at"] end end diff --git a/app/serializers/export/comment_serializer.rb b/app/serializers/export/comment_serializer.rb new file mode 100644 index 000000000..72e9ccbf5 --- /dev/null +++ b/app/serializers/export/comment_serializer.rb @@ -0,0 +1,10 @@ +module Export + class CommentSerializer < ActiveModel::Serializer + attributes :text, + :post_guid + + def post_guid + object.post.guid + end + end +end diff --git a/app/serializers/export/post_serializer.rb b/app/serializers/export/post_serializer.rb new file mode 100644 index 000000000..3459c2a51 --- /dev/null +++ b/app/serializers/export/post_serializer.rb @@ -0,0 +1,14 @@ +module Export + class PostSerializer < ActiveModel::Serializer + attributes :text, + :public, + :diaspora_handle, + :type, + :image_url, + :image_height, + :image_width, + :likes_count, + :comments_count, + :reshares_count + end +end diff --git a/app/serializers/export/user_serializer.rb b/app/serializers/export/user_serializer.rb index da277dc7f..9621d2095 100644 --- a/app/serializers/export/user_serializer.rb +++ b/app/serializers/export/user_serializer.rb @@ -11,6 +11,12 @@ module Export has_one :profile, serializer: Export::ProfileSerializer has_many :aspects, each_serializer: Export::AspectSerializer has_many :contacts, each_serializer: Export::ContactSerializer + has_many :posts, each_serializer: Export::PostSerializer + has_many :comments, each_serializer: Export::CommentSerializer + + def comments + object.person.comments + end end end \ No newline at end of file diff --git a/app/uploaders/exported_user.rb b/app/uploaders/exported_user.rb new file mode 100644 index 000000000..ff323ecce --- /dev/null +++ b/app/uploaders/exported_user.rb @@ -0,0 +1,19 @@ +# Copyright (c) 2010-2011, Diaspora Inc. This file is +# licensed under the Affero General Public License version 3 or later. See +# the COPYRIGHT file. + +class ExportedUser < CarrierWave::Uploader::Base + + def store_dir + "uploads/users" + end + + def extension_white_list + %w(gz) + end + + def filename + "#{model.username}_diaspora_data.json.gz" + end + +end diff --git a/app/views/users/edit.html.haml b/app/views/users/edit.html.haml index 597979811..38645adcf 100644 --- a/app/views/users/edit.html.haml +++ b/app/views/users/edit.html.haml @@ -180,8 +180,20 @@ #account_data.span6 %h3 = t('.export_data') - .small-horizontal-spacer - = link_to t('.download_profile'), export_user_path(format: :json), :class => "button" + - if current_user.exporting + .small-horizontal-spacer + .export-in-progress= t('.export_in_progress') + - elsif current_user.export.present? + .small-horizontal-spacer + = link_to t('.download_export'), download_profile_user_path, class: "button" + .small-horizontal-spacer + = t('.last_exported_at', timestamp: current_user.exported_at) + .small-horizontal-spacer + = link_to t('.request_export_update'), export_profile_user_path + - else + .small-horizontal-spacer + = link_to t('.request_export'), export_profile_user_path, :class => "button" + .small-horizontal-spacer = link_to t('.download_photos'), "#", :class => "button", :id => "photo-export-button", :title => t('.photo_export_unavailable') diff --git a/app/views/users/edit.mobile.haml b/app/views/users/edit.mobile.haml index d71ce824a..c26390eb8 100644 --- a/app/views/users/edit.mobile.haml +++ b/app/views/users/edit.mobile.haml @@ -161,7 +161,19 @@ #account_data.span-5.append-2 %h4 = t('.export_data') - = link_to t('.download_xml'), export_user_path, :class => "btn" + - if current_user.exporting + .small-horizontal-spacer + .export-in-progress= t('.export_in_progress') + - elsif current_user.export.present? + .small-horizontal-spacer + = link_to t('.download_export'), download_profile_user_path, class: "button" + .small-horizontal-spacer + = t('.last_exported_at', timestamp: current_user.exported_at) + .small-horizontal-spacer + = link_to t('.request_export_update'), export_profile_user_path + - else + .small-horizontal-spacer + = link_to t('.request_export'), export_profile_user_path, :class => "button" %br %br = link_to t('.download_photos'), "#", :class => "btn", :id => "photo-export-button", :title => t('.photo_export_unavailable') diff --git a/app/views/users/export_email.markerb b/app/views/users/export_email.markerb new file mode 100644 index 000000000..c6a9c156c --- /dev/null +++ b/app/views/users/export_email.markerb @@ -0,0 +1 @@ +<%= t('notifier.export_email.body', url: @user.export.url, username: @user.username) %> diff --git a/app/views/users/export_failure_email.markerb b/app/views/users/export_failure_email.markerb new file mode 100644 index 000000000..64dd314ad --- /dev/null +++ b/app/views/users/export_failure_email.markerb @@ -0,0 +1 @@ +<%= t('notifier.export_failure_email.body', username: @user.username) %> diff --git a/app/workers/export_user.rb b/app/workers/export_user.rb new file mode 100644 index 000000000..f01535ce9 --- /dev/null +++ b/app/workers/export_user.rb @@ -0,0 +1,21 @@ +# Copyright (c) 2010-2011, Diaspora Inc. This file is +# licensed under the Affero General Public License version 3 or later. See +# the COPYRIGHT file. + + +module Workers + class ExportUser < Base + sidekiq_options queue: :export_user + + def perform(user_id) + @user = User.find(user_id) + @user.perform_export! + + if @user.reload.export.present? + ExportMailer.export_complete_for(@user) + else + ExportMailer.export_failure_for(@user) + end + end + end +end diff --git a/config/locales/diaspora/en.yml b/config/locales/diaspora/en.yml index 6ab014aaa..80f13fda3 100644 --- a/config/locales/diaspora/en.yml +++ b/config/locales/diaspora/en.yml @@ -781,6 +781,27 @@ en: The diaspora* email robot! [1]: %{url} + export_email: + subject: "Your personal data is ready for download, %{username}" + body: |- + Hello %{username} + + Your data has been processed and is ready for download by following [this link][%{url}]. + + Cheers, + + The diaspora* email robot! + export_failure_email: + subject: "We're sorry, there was an issue with your data, %{username}" + body: |- + Hello %{username} + + We''ve encountered an issue while processing your personal data for download. + Please try again! + + Cheers, + + The diaspora* email robot! accept_invite: "Accept Your diaspora* invite!" invited_you: "%{name} invited you to diaspora*" invite: @@ -1254,7 +1275,11 @@ en: current_password: "Current password" current_password_expl: "the one you sign in with..." character_minimum_expl: "must be at least six characters" - download_profile: "download my profile" + export_in_progress: 'We are currently processing your data. Please check back in a few moments.' + last_exported_at: '(Last updated at %{timestamp})' + request_export: 'request my profile data' + download_export: 'download my profile' + request_export_update: 'refresh my profile data' download_photos: "download my photos" your_handle: "Your diaspora* ID" your_email: "Your email" diff --git a/config/routes.rb b/config/routes.rb index ce1211a97..7d6e9785f 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -101,7 +101,8 @@ Diaspora::Application.routes.draw do resource :user, :only => [:edit, :update, :destroy], :shallow => true do get :getting_started_completed - get :export, format: :json + get :export_profile + get :download_profile get :export_photos end diff --git a/db/migrate/20141227120907_add_export_to_user.rb b/db/migrate/20141227120907_add_export_to_user.rb new file mode 100644 index 000000000..376d5fe36 --- /dev/null +++ b/db/migrate/20141227120907_add_export_to_user.rb @@ -0,0 +1,7 @@ +class AddExportToUser < ActiveRecord::Migration + def change + add_column :users, :export, :string + add_column :users, :exported_at, :datetime + add_column :users, :exporting, :boolean, default: false + end +end diff --git a/db/schema.rb b/db/schema.rb index b5112c5e8..d176940cd 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20141216213423) do +ActiveRecord::Schema.define(version: 20141227120907) do create_table "account_deletions", force: true do |t| t.string "diaspora_handle" @@ -558,6 +558,9 @@ ActiveRecord::Schema.define(version: 20141216213423) do t.datetime "reset_password_sent_at" t.datetime "last_seen" t.datetime "remove_after" + t.string "export" + t.datetime "exported_at" + t.boolean "exporting", default: false end add_index "users", ["authentication_token"], name: "index_users_on_authentication_token", unique: true, using: :btree diff --git a/spec/controllers/users_controller_spec.rb b/spec/controllers/users_controller_spec.rb index a2bc3d511..82a3561a8 100644 --- a/spec/controllers/users_controller_spec.rb +++ b/spec/controllers/users_controller_spec.rb @@ -11,10 +11,21 @@ describe UsersController, :type => :controller do allow(@controller).to receive(:current_user).and_return(@user) end - describe '#export' do - it 'can return a json file' do - get :export, format: :json - expect(response.header["Content-Type"]).to include "application/json" + describe '#export_profile' do + it 'queues an export job' do + expect(@user).to receive :queue_export + get :export_profile + expect(request.flash[:notice]).to eql(I18n.t('users.edit.export_in_progress')) + expect(response).to redirect_to(edit_user_path) + end + end + + describe "#download_profile" do + it "downloads a user's export file" do + @user.perform_export! + get :download_profile + parsed = JSON.parse(ActiveSupport::Gzip.decompress(response.body)) + expect(parsed['user']['username']).to eq @user.username end end diff --git a/spec/mailers/export_spec.rb b/spec/mailers/export_spec.rb new file mode 100644 index 000000000..b699c10d5 --- /dev/null +++ b/spec/mailers/export_spec.rb @@ -0,0 +1,37 @@ +# Copyright (c) 2010-2011, Diaspora Inc. This file is +# licensed under the Affero General Public License version 3 or later. See +# the COPYRIGHT file. + +require 'spec_helper' + +describe ExportMailer, :type => :mailer do + describe '#export_complete_for' do + it "should deliver successfully" do + expect { ExportMailer.export_complete_for(alice) }.to_not raise_error + end + + it "should be added to the delivery queue" do + expect { ExportMailer.export_complete_for(alice) }.to change(ActionMailer::Base.deliveries, :size).by(1) + end + + it "should include correct recipient" do + ExportMailer.export_complete_for(alice) + expect(ActionMailer::Base.deliveries[0].to[0]).to include(alice.email) + end + end + + describe '#export_failure_for' do + it "should deliver successfully" do + expect { ExportMailer.export_failure_for(alice) }.to_not raise_error + end + + it "should be added to the delivery queue" do + expect { ExportMailer.export_failure_for(alice) }.to change(ActionMailer::Base.deliveries, :size).by(1) + end + + it "should include correct recipient" do + ExportMailer.export_failure_for(alice) + expect(ActionMailer::Base.deliveries[0].to[0]).to include(alice.email) + end + end +end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 4648b3196..33be531b0 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -996,6 +996,33 @@ describe User, :type => :model do end end + describe "queue_export" do + it "queues up a job to perform the export" do + user = FactoryGirl.create :user + expect(Workers::ExportUser).to receive(:perform_async).with(user.id) + user.queue_export + expect(user.exporting).to be_truthy + end + end + + describe "perform_export!" do + it "saves a json export to the user" do + user = FactoryGirl.create :user, exporting: true + user.perform_export! + expect(user.export).to be_present + expect(user.exported_at).to be_present + expect(user.exporting).to be_falsey + expect(user.export.filename).to match /.json/ + expect(ActiveSupport::Gzip.decompress(user.export.file.read)).to include user.username + end + + it "compresses the result" do + user = FactoryGirl.create :user, exporting: true + expect(ActiveSupport::Gzip).to receive :compress + user.perform_export! + end + end + describe "sign up" do before do params = {:username => "ohai", diff --git a/spec/workers/export_user_spec.rb b/spec/workers/export_user_spec.rb new file mode 100644 index 000000000..f6ecfba55 --- /dev/null +++ b/spec/workers/export_user_spec.rb @@ -0,0 +1,26 @@ +require 'spec_helper' + +describe Workers::ExportUser do + + before do + allow(User).to receive(:find).with(alice.id).and_return(alice) + end + + it 'calls export! on user with given id' do + expect(alice).to receive(:perform_export!) + Workers::ExportUser.new.perform(alice.id) + end + + it 'sends a success message when the export is successful' do + alice.stub(:export).and_return(OpenStruct.new) + expect(ExportMailer).to receive(:export_complete_for).with(alice) + Workers::ExportUser.new.perform(alice.id) + end + + it 'sends a failure message when the export fails' do + alice.stub(:export).and_return(nil) + expect(alice).to receive(:perform_export!).and_return(false) + expect(ExportMailer).to receive(:export_failure_for).with(alice) + Workers::ExportUser.new.perform(alice.id) + end +end