Add contacts/posts, and GZipping JSON exporter output

Changelog.md

@@ -137,12 +137,12 @@ diaspora.yml file**. The existing settings from 0.4.x and before will not work a
 * Truncate too long OpenGraph descriptions [#5387](https://github.com/diaspora/diaspora/pull/5387)
 * Make the source code URL configurable [#5410](https://github.com/diaspora/diaspora/pull/5410)
 * Prefill publisher on the tag pages [#5442](https://github.com/diaspora/diaspora/pull/5442)
-* Allows users to export their data in JSON format from their user settings page [#5354](https://github.com/diaspora/diaspora/pull/5354)
 * Don't include the content of non-public posts into notification mails [#5494](https://github.com/diaspora/diaspora/pull/5494)
 * Allow to set unhosted button and currency for paypal donation [#5452](https://github.com/diaspora/diaspora/pull/5452)
 * Add followed tags in the mobile menu [#5468](https://github.com/diaspora/diaspora/pull/5468)
 * Replace Pagedown with markdown-it [#5526](https://github.com/diaspora/diaspora/pull/5526)
 * Do not truncate notification emails anymore [#4342](https://github.com/diaspora/diaspora/issues/4342)
+* Allows users to export their data in gzipped JSON format from their user settings page [#5499](https://github.com/diaspora/diaspora/pull/5499)
 
 # 0.4.1.2
 

app/controllers/users_controller.rb

@@ -135,12 +135,14 @@ class UsersController < ApplicationController
     redirect_to stream_path
   end
 
-  def export
-    if export = Diaspora::Exporter.new(current_user).execute
-      send_data export, filename: "#{current_user.username}_diaspora_data.json", type: :json
-    else
-      head :not_acceptable
-    end
+  def export_profile
+    current_user.queue_export
+    flash[:notice] = I18n.t('users.edit.export_in_progress')
+    redirect_to edit_user_path
+  end
+
+  def download_profile
+    send_data File.open(current_user.export.path).read, type: :json, filename: current_user.export.filename
   end
 
   def export_photos

app/mailers/export_mailer.rb

@@ -0,0 +1,22 @@
+class ExportMailer < ActionMailer::Base
+  default from: AppConfig.mail.sender_address
+
+  def export_complete_for(user)
+    @user = user
+
+    mail(to: @user.email, subject: I18n.t('notifier.export_email.subject')) do |format|
+      format.html { render 'users/export_email' }
+      format.text { render 'users/export_email' }
+    end.deliver
+  end
+
+  def export_failure_for(user)
+    @user = user
+
+    mail(to: @user.email, subject: I18n.t('notifier.export_failure_email.subject')) do |format|
+      format.html { render 'users/export_failure_email' }
+      format.text { render 'users/export_failure_email' }
+    end.deliver
+  end
+
+end

app/models/user.rb

@@ -291,6 +291,28 @@ class User < ActiveRecord::Base
     end
   end
 
+  ######### Data export ##################
+  mount_uploader :export, ExportedUser
+
+  def queue_export
+    update exporting: true
+    Workers::ExportUser.perform_async(id)
+  end
+
+  def perform_export!
+    export = Tempfile.new([username, '.json.gz'], encoding: 'ascii-8bit')
+    export.write(compressed_export) && export.close
+    if export.present?
+      update exporting: false, export: export, exported_at: Time.zone.now
+    else
+      update exporting: false
+    end
+  end
+
+  def compressed_export
+    ActiveSupport::Gzip.compress Diaspora::Exporter.new(self).execute
+  end
+
   ######### Mailer #######################
   def mail(job, *args)
     pref = job.to_s.gsub('Workers::Mail::', '').underscore
@@ -505,6 +527,6 @@ class User < ActiveRecord::Base
                             "created_at", "updated_at", "locked_at",
                             "serialized_private_key", "getting_started",
                             "disable_mail", "show_community_spotlight_in_stream",
-                            "email", "remove_after"]
+                            "email", "remove_after", "export", "exporting", "exported_at"]
   end
 end

app/serializers/export/comment_serializer.rb

@@ -0,0 +1,10 @@
+module Export
+  class CommentSerializer < ActiveModel::Serializer
+    attributes :text,
+               :post_guid
+
+    def post_guid
+      object.post.guid
+    end
+  end
+end

app/serializers/export/post_serializer.rb

@@ -0,0 +1,14 @@
+module Export
+  class PostSerializer < ActiveModel::Serializer
+    attributes :text,
+               :public,
+               :diaspora_handle,
+               :type,
+               :image_url,
+               :image_height,
+               :image_width,
+               :likes_count,
+               :comments_count,
+               :reshares_count
+  end
+end

app/serializers/export/user_serializer.rb

@@ -11,6 +11,12 @@ module Export
     has_one    :profile,  serializer:      Export::ProfileSerializer
     has_many   :aspects,  each_serializer: Export::AspectSerializer
     has_many   :contacts, each_serializer: Export::ContactSerializer
+    has_many   :posts,    each_serializer: Export::PostSerializer
+    has_many   :comments, each_serializer: Export::CommentSerializer
+
+    def comments
+      object.person.comments
+    end
 
   end
 end

app/uploaders/exported_user.rb

@@ -0,0 +1,19 @@
+#   Copyright (c) 2010-2011, Diaspora Inc.  This file is
+#   licensed under the Affero General Public License version 3 or later.  See
+#   the COPYRIGHT file.
+
+class ExportedUser < CarrierWave::Uploader::Base
+
+  def store_dir
+    "uploads/users"
+  end
+
+  def extension_white_list
+    %w(gz)
+  end
+
+  def filename
+    "#{model.username}_diaspora_data.json.gz"
+  end
+
+end

app/views/users/edit.html.haml

@@ -180,8 +180,20 @@
         #account_data.span6
           %h3 
             = t('.export_data')
-          .small-horizontal-spacer          
-          = link_to t('.download_profile'), export_user_path(format: :json), :class => "button"
+          - if current_user.exporting
+            .small-horizontal-spacer
+              .export-in-progress= t('.export_in_progress')
+          - elsif current_user.export.present?
+            .small-horizontal-spacer
+              = link_to t('.download_export'), download_profile_user_path, class: "button"
+            .small-horizontal-spacer
+              = t('.last_exported_at', timestamp: current_user.exported_at)
+            .small-horizontal-spacer
+              = link_to t('.request_export_update'), export_profile_user_path
+          - else
+            .small-horizontal-spacer
+              = link_to t('.request_export'), export_profile_user_path, :class => "button"
+
           .small-horizontal-spacer
           = link_to t('.download_photos'), "#", :class => "button", :id => "photo-export-button", :title => t('.photo_export_unavailable')
 

app/views/users/edit.mobile.haml

@@ -161,7 +161,19 @@
   #account_data.span-5.append-2
     %h4
       = t('.export_data')
-    = link_to t('.download_xml'), export_user_path, :class => "btn"
+    - if current_user.exporting
+      .small-horizontal-spacer
+        .export-in-progress= t('.export_in_progress')
+    - elsif current_user.export.present?
+      .small-horizontal-spacer
+        = link_to t('.download_export'), download_profile_user_path, class: "button"
+      .small-horizontal-spacer
+        = t('.last_exported_at', timestamp: current_user.exported_at)
+      .small-horizontal-spacer
+        = link_to t('.request_export_update'), export_profile_user_path
+    - else
+      .small-horizontal-spacer
+        = link_to t('.request_export'), export_profile_user_path, :class => "button"
     %br
     %br
     = link_to t('.download_photos'), "#", :class => "btn", :id => "photo-export-button", :title => t('.photo_export_unavailable')

app/views/users/export_email.markerb

@@ -0,0 +1 @@
+<%= t('notifier.export_email.body', url: @user.export.url, username: @user.username) %>

app/views/users/export_failure_email.markerb

@@ -0,0 +1 @@
+<%= t('notifier.export_failure_email.body', username: @user.username) %>

app/workers/export_user.rb

@@ -0,0 +1,21 @@
+#   Copyright (c) 2010-2011, Diaspora Inc.  This file is
+#   licensed under the Affero General Public License version 3 or later.  See
+#   the COPYRIGHT file.
+
+
+module Workers
+  class ExportUser < Base
+    sidekiq_options queue: :export_user
+
+    def perform(user_id)
+      @user = User.find(user_id)
+      @user.perform_export!
+
+      if @user.reload.export.present?
+        ExportMailer.export_complete_for(@user)
+      else
+        ExportMailer.export_failure_for(@user)
+      end
+    end
+  end
+end

config/locales/diaspora/en.yml

@@ -781,6 +781,27 @@ en:
           The diaspora* email robot!
 
           [1]: %{url}
+    export_email:
+      subject: "Your personal data is ready for download, %{username}"
+      body: |-
+        Hello %{username}
+
+        Your data has been processed and is ready for download by following [this link][%{url}].
+
+        Cheers,
+
+        The diaspora* email robot!
+    export_failure_email:
+      subject: "We're sorry, there was an issue with your data, %{username}"
+      body: |-
+        Hello %{username}
+
+        We''ve encountered an issue while processing your personal data for download.
+        Please try again!
+
+        Cheers,
+
+        The diaspora* email robot!
     accept_invite: "Accept Your diaspora* invite!"
     invited_you: "%{name} invited you to diaspora*"
     invite:
@@ -1254,7 +1275,11 @@ en:
       current_password: "Current password"
       current_password_expl: "the one you sign in with..."
       character_minimum_expl: "must be at least six characters"
-      download_profile: "download my profile"
+      export_in_progress: 'We are currently processing your data. Please check back in a few moments.'
+      last_exported_at: '(Last updated at %{timestamp})'
+      request_export: 'request my profile data'
+      download_export: 'download my profile'
+      request_export_update: 'refresh my profile data'
       download_photos: "download my photos"
       your_handle: "Your diaspora* ID"
       your_email: "Your email"

config/routes.rb

@@ -101,7 +101,8 @@ Diaspora::Application.routes.draw do
 
   resource :user, :only => [:edit, :update, :destroy], :shallow => true do
     get :getting_started_completed
-    get :export, format: :json
+    get :export_profile
+    get :download_profile
     get :export_photos
   end
 

db/migrate/20141227120907_add_export_to_user.rb

@@ -0,0 +1,7 @@
+class AddExportToUser < ActiveRecord::Migration
+  def change
+    add_column :users, :export, :string
+    add_column :users, :exported_at, :datetime
+    add_column :users, :exporting, :boolean, default: false
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20141216213423) do
+ActiveRecord::Schema.define(version: 20141227120907) do
 
   create_table "account_deletions", force: true do |t|
     t.string   "diaspora_handle"
@@ -558,6 +558,9 @@ ActiveRecord::Schema.define(version: 20141216213423) do
     t.datetime "reset_password_sent_at"
     t.datetime "last_seen"
     t.datetime "remove_after"
+    t.string   "export"
+    t.datetime "exported_at"
+    t.boolean  "exporting",                                      default: false
   end
 
   add_index "users", ["authentication_token"], name: "index_users_on_authentication_token", unique: true, using: :btree

spec/controllers/users_controller_spec.rb

@@ -11,10 +11,21 @@ describe UsersController, :type => :controller do
     allow(@controller).to receive(:current_user).and_return(@user)
   end
 
-  describe '#export' do
-    it 'can return a json file' do
-      get :export, format: :json
-      expect(response.header["Content-Type"]).to include "application/json"
+  describe '#export_profile' do
+    it 'queues an export job' do
+      expect(@user).to receive :queue_export
+      get :export_profile
+      expect(request.flash[:notice]).to eql(I18n.t('users.edit.export_in_progress'))
+      expect(response).to redirect_to(edit_user_path)
+    end
+  end
+
+  describe "#download_profile" do
+    it "downloads a user's export file" do
+      @user.perform_export!
+      get :download_profile
+      parsed = JSON.parse(ActiveSupport::Gzip.decompress(response.body))
+      expect(parsed['user']['username']).to eq @user.username
     end
   end
 

spec/mailers/export_spec.rb

@@ -0,0 +1,37 @@
+#   Copyright (c) 2010-2011, Diaspora Inc.  This file is
+#   licensed under the Affero General Public License version 3 or later.  See
+#   the COPYRIGHT file.
+
+require 'spec_helper'
+
+describe ExportMailer, :type => :mailer do
+  describe '#export_complete_for' do
+    it "should deliver successfully" do
+      expect { ExportMailer.export_complete_for(alice) }.to_not raise_error
+    end
+
+    it "should be added to the delivery queue" do
+      expect { ExportMailer.export_complete_for(alice) }.to change(ActionMailer::Base.deliveries, :size).by(1)
+    end
+
+    it "should include correct recipient" do
+      ExportMailer.export_complete_for(alice)
+      expect(ActionMailer::Base.deliveries[0].to[0]).to include(alice.email)
+    end
+  end
+
+  describe '#export_failure_for' do
+    it "should deliver successfully" do
+      expect { ExportMailer.export_failure_for(alice) }.to_not raise_error
+    end
+
+    it "should be added to the delivery queue" do
+      expect { ExportMailer.export_failure_for(alice) }.to change(ActionMailer::Base.deliveries, :size).by(1)
+    end
+
+    it "should include correct recipient" do
+      ExportMailer.export_failure_for(alice)
+      expect(ActionMailer::Base.deliveries[0].to[0]).to include(alice.email)
+    end
+  end
+end

spec/models/user_spec.rb

@@ -996,6 +996,33 @@ describe User, :type => :model do
     end
   end
 
+  describe "queue_export" do
+    it "queues up a job to perform the export" do
+      user = FactoryGirl.create :user
+      expect(Workers::ExportUser).to receive(:perform_async).with(user.id)
+      user.queue_export
+      expect(user.exporting).to be_truthy
+    end
+  end
+
+  describe "perform_export!" do
+    it "saves a json export to the user" do
+      user = FactoryGirl.create :user, exporting: true
+      user.perform_export!
+      expect(user.export).to be_present
+      expect(user.exported_at).to be_present
+      expect(user.exporting).to be_falsey
+      expect(user.export.filename).to match /.json/
+      expect(ActiveSupport::Gzip.decompress(user.export.file.read)).to include user.username
+    end
+
+    it "compresses the result" do
+      user = FactoryGirl.create :user, exporting: true
+      expect(ActiveSupport::Gzip).to receive :compress
+      user.perform_export!
+    end
+  end
+
   describe "sign up" do
     before do
       params = {:username => "ohai",

spec/workers/export_user_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe Workers::ExportUser do
+
+  before do
+    allow(User).to receive(:find).with(alice.id).and_return(alice)
+  end
+
+  it 'calls export! on user with given id' do
+    expect(alice).to receive(:perform_export!)
+    Workers::ExportUser.new.perform(alice.id)
+  end
+
+  it 'sends a success message when the export is successful' do
+    alice.stub(:export).and_return(OpenStruct.new)
+    expect(ExportMailer).to receive(:export_complete_for).with(alice)
+    Workers::ExportUser.new.perform(alice.id)
+  end
+
+  it 'sends a failure message when the export fails' do
+    alice.stub(:export).and_return(nil)
+    expect(alice).to receive(:perform_export!).and_return(false)
+    expect(ExportMailer).to receive(:export_failure_for).with(alice)
+    Workers::ExportUser.new.perform(alice.id)
+  end
+end