Merge pull request #8260 from tclaus/import_zipped_archive

Migration: Import compressed archive file handling

Changelog.md

@@ -32,7 +32,7 @@ Although the chat was never enabled per default and was marked as experimental,
 ## Features
 * Add client-side cropping of profile image uploads [#7581](https://github.com/diaspora/diaspora/pull/7581)
 * Add client-site rescaling of post images if they exceed the maximum possible size [#7734](https://github.com/diaspora/diaspora/pull/7734)
-* Add backend for archive import [#7660](https://github.com/diaspora/diaspora/pull/7660) [#8254](https://github.com/diaspora/diaspora/pull/8254) [#8264](https://github.com/diaspora/diaspora/pull/8264) [#8010](https://github.com/diaspora/diaspora/pull/8010)
+* Add backend for archive import [#7660](https://github.com/diaspora/diaspora/pull/7660) [#8254](https://github.com/diaspora/diaspora/pull/8254) [#8264](https://github.com/diaspora/diaspora/pull/8264) [#8010](https://github.com/diaspora/diaspora/pull/8010) [#8260](https://github.com/diaspora/diaspora/pull/8260)
 * For pods running PostgreSQL, make sure that no upper-case/mixed-case tags exist, and create a `lower(name)` index on tags to speed up ActsAsTaggableOn [#8206](https://github.com/diaspora/diaspora/pull/8206)
 * Allow podmins/moderators to see all local public posts to improve moderation [#8232](https://github.com/diaspora/diaspora/pull/8232)
 

app/services/migration_service.rb

@@ -19,6 +19,8 @@ class MigrationService
     find_or_create_user
     import_archive
     run_migration
+  ensure
+    remove_intermediate_file
   end
 
   # when old person can't be resolved we still import data but we don't create&perform AccountMigration instance
@@ -64,10 +66,56 @@ class MigrationService
   end
 
   def archive_file
-    # TODO: archive is likely to be a .json.gz file
+    return uncompressed_zip if zip_file?
+    return uncompressed_gz if gzip_file?
+
     File.new(archive_path, "r")
   end
 
+  def zip_file?
+    filetype = MIME::Types.type_for(archive_path).first.content_type
+    filetype.eql?("application/zip")
+  end
+
+  def gzip_file?
+    filetype = MIME::Types.type_for(archive_path).first.content_type
+    filetype.eql?("application/gzip")
+  end
+
+  def uncompressed_gz
+    target_dir = File.dirname(archive_path) + Pathname::SEPARATOR_LIST
+    unzipped_archive_file = "#{File.join(target_dir, SecureRandom.hex)}.json" # never override an existing file
+
+    Zlib::GzipReader.open(archive_path) {|gz|
+      File.open(unzipped_archive_file, "w") do |output_stream|
+        IO.copy_stream(gz, output_stream)
+      end
+      @intermediate_file = unzipped_archive_file
+    }
+    File.new(unzipped_archive_file, "r")
+  end
+
+  def uncompressed_zip
+    target_dir = File.dirname(archive_path) + Pathname::SEPARATOR_LIST
+    zip_stream = Zip::InputStream.open(archive_path)
+    while entry = zip_stream.get_next_entry # rubocop:disable Lint/AssignmentInCondition
+      next unless entry.name.end_with?(".json")
+
+      target_file = "#{File.join(target_dir, SecureRandom.hex)}.json" # never override an existing file
+      entry.extract(target_file)
+      @intermediate_file = target_file
+      return File.new(target_file, "r")
+    end
+  end
+
+  def remove_intermediate_file
+    # If an unzip operation created an unzipped file, remove it after migration
+    return if @intermediate_file.nil?
+    return unless File.exist?(@intermediate_file)
+
+    File.delete(@intermediate_file)
+  end
+
   class ArchiveValidationFailed < RuntimeError
   end
 

spec/integration/migration_service_spec.rb

@@ -188,7 +188,7 @@ describe MigrationService do
   let(:new_username) { "newuser" }
   let(:new_user_handle) { "#{new_username}@#{AppConfig.bare_pod_uri}" }
 
-  let(:archive_file) { Tempfile.new("archive") }
+  let(:archive_file) { Tempfile.new(%w[archive .json]) }
 
   def setup_validation_time_expectations
     expect_person_fetch(contact2_diaspora_id, nil)
@@ -314,6 +314,49 @@ describe MigrationService do
     end
   end
 
+  context "compressed archives" do
+    it "uncompresses gz archive" do
+      gz_compressed_file = create_gz_archive
+      service = MigrationService.new(gz_compressed_file, new_username)
+      uncompressed_file = service.send(:archive_file)
+      json = uncompressed_file.read
+      expect {
+        JSON.parse(json)
+      }.not_to raise_error
+    end
+
+    it "uncompresses zip archive" do
+      zip_compressed_file = create_zip_archive
+      service = MigrationService.new(zip_compressed_file, new_username)
+      uncompressed_file = service.send(:archive_file)
+      json = uncompressed_file.read
+      expect {
+        JSON.parse(json)
+      }.not_to raise_error
+    end
+
+    def create_gz_archive
+      target_file = Tempfile.new(%w[archive .json.gz]).path
+      Zlib::GzipWriter.open(target_file) do |gz|
+        File.open(archive_file.path).each do |line|
+          gz.write line
+        end
+      end
+      target_file
+    end
+
+    def create_zip_archive
+      target_file = Tempfile.new(%w[archive .zip]).path
+      Zip::OutputStream.open(target_file) do |zip|
+        zip.put_next_entry("archive.json")
+        File.open(archive_file.path).each do |line|
+          zip.write line
+        end
+      end
+      target_file
+    end
+  end
+
   context "old user is a known remote user" do
     let(:old_person) {
       FactoryBot.create(:person,