Import compressed archive file

This commit is contained in:
Thorsten Claus 2021-06-27 12:09:26 +02:00 committed by Benjamin Neff
parent 8d5abe8892
commit 85a5744867
No known key found for this signature in database
GPG key ID: 971464C3F1A90194
2 changed files with 93 additions and 2 deletions

View file

@ -19,6 +19,8 @@ class MigrationService
find_or_create_user find_or_create_user
import_archive import_archive
run_migration run_migration
ensure
remove_intermediate_file
end end
# when old person can't be resolved we still import data but we don't create&perform AccountMigration instance # when old person can't be resolved we still import data but we don't create&perform AccountMigration instance
@ -64,10 +66,56 @@ class MigrationService
end end
def archive_file def archive_file
# TODO: archive is likely to be a .json.gz file return uncompressed_zip if zip_file?
return uncompressed_gz if gzip_file?
File.new(archive_path, "r") File.new(archive_path, "r")
end end
def zip_file?
filetype = MIME::Types.type_for(archive_path).first.content_type
filetype.eql?("application/zip")
end
def gzip_file?
filetype = MIME::Types.type_for(archive_path).first.content_type
filetype.eql?("application/gzip")
end
def uncompressed_gz
target_dir = File.dirname(archive_path) + Pathname::SEPARATOR_LIST
unzipped_archive_file = "#{File.join(target_dir, SecureRandom.hex)}.json" # never override an existing file
Zlib::GzipReader.open(archive_path) {|gz|
File.open(unzipped_archive_file, "w") do |output_stream|
IO.copy_stream(gz, output_stream)
end
@intermediate_file = unzipped_archive_file
}
File.new(unzipped_archive_file, "r")
end
def uncompressed_zip
target_dir = File.dirname(archive_path) + Pathname::SEPARATOR_LIST
zip_stream = Zip::InputStream.open(archive_path)
while entry = zip_stream.get_next_entry # rubocop:disable Lint/AssignmentInCondition
next unless entry.name.end_with?(".json")
target_file = "#{File.join(target_dir, SecureRandom.hex)}.json" # never override an existing file
entry.extract(target_file)
@intermediate_file = target_file
return File.new(target_file, "r")
end
end
def remove_intermediate_file
# If an unzip operation created an unzipped file, remove it after migration
return if @intermediate_file.nil?
return unless File.exist?(@intermediate_file)
File.delete(@intermediate_file)
end
class ArchiveValidationFailed < RuntimeError class ArchiveValidationFailed < RuntimeError
end end

View file

@ -188,7 +188,7 @@ describe MigrationService do
let(:new_username) { "newuser" } let(:new_username) { "newuser" }
let(:new_user_handle) { "#{new_username}@#{AppConfig.bare_pod_uri}" } let(:new_user_handle) { "#{new_username}@#{AppConfig.bare_pod_uri}" }
let(:archive_file) { Tempfile.new("archive") } let(:archive_file) { Tempfile.new(%w[archive .json]) }
def setup_validation_time_expectations def setup_validation_time_expectations
expect_person_fetch(contact2_diaspora_id, nil) expect_person_fetch(contact2_diaspora_id, nil)
@ -314,6 +314,49 @@ describe MigrationService do
end end
end end
context "compressed archives" do
it "uncompresses gz archive" do
gz_compressed_file = create_gz_archive
service = MigrationService.new(gz_compressed_file, new_username)
uncompressed_file = service.send(:archive_file)
json = uncompressed_file.read
expect {
JSON.parse(json)
}.not_to raise_error
end
it "uncompresses zip archive" do
zip_compressed_file = create_zip_archive
service = MigrationService.new(zip_compressed_file, new_username)
uncompressed_file = service.send(:archive_file)
json = uncompressed_file.read
expect {
JSON.parse(json)
}.not_to raise_error
end
def create_gz_archive
target_file = Tempfile.new(%w[archive .json.gz]).path
Zlib::GzipWriter.open(target_file) do |gz|
File.open(archive_file.path).each do |line|
gz.write line
end
end
target_file
end
def create_zip_archive
target_file = Tempfile.new(%w[archive .zip]).path
Zip::OutputStream.open(target_file) do |zip|
zip.put_next_entry("archive.json")
File.open(archive_file.path).each do |line|
zip.write line
end
end
target_file
end
end
context "old user is a known remote user" do context "old user is a known remote user" do
let(:old_person) { let(:old_person) {
FactoryBot.create(:person, FactoryBot.create(:person,