More logging (w/ elapsed time) on mysql export.
This commit is contained in:
Sarah Mei
parent
c643346589
commit
a4c01e4df1
2 changed files with 45 additions and 12 deletions
|
|
@ -1,9 +1,17 @@
|
|||
# Copyright (c) 2010, Diaspora Inc. This file is
|
||||
# licensed under the Affero General Public License version 3 or later. See
|
||||
# the COPYRIGHT file.
|
||||
|
||||
require 'json'
|
||||
require 'csv'
|
||||
|
||||
class MongoToMysql
|
||||
attr_accessor :start_time
|
||||
|
||||
def initialize(start_time = Time.now)
|
||||
@start_time = start_time
|
||||
end
|
||||
|
||||
def csv_options
|
||||
{:col_sep => ",",
|
||||
:row_sep => :auto,
|
||||
|
|
@ -17,20 +25,25 @@ class MongoToMysql
|
|||
:skip_blanks => false,
|
||||
:force_quotes => false }
|
||||
end
|
||||
|
||||
def dirname
|
||||
"tmp/export-for-mysql"
|
||||
end
|
||||
|
||||
def dirpath
|
||||
"#{Rails.root}/#{dirname}"
|
||||
end
|
||||
|
||||
def clear_dir
|
||||
`rm -rf #{dirpath}`
|
||||
`mkdir -p #{dirpath}/json`
|
||||
`mkdir -p #{dirpath}/csv`
|
||||
end
|
||||
|
||||
def db_name
|
||||
"diaspora-#{Rails.env}"
|
||||
end
|
||||
|
||||
def models
|
||||
@models ||= [ {:name => :aspects},
|
||||
{:name => :comments},
|
||||
|
|
@ -43,40 +56,50 @@ class MongoToMysql
|
|||
{:name => :users},
|
||||
]
|
||||
end
|
||||
|
||||
def id_sed
|
||||
@id_sed = sed_replace('{\ \"$oid\"\ :\ \(\"[^"]*\"\)\ }')
|
||||
end
|
||||
|
||||
def date_sed
|
||||
@date_sed = sed_replace('{\ \"$date\"\ :\ \([0-9]*\)\ }')
|
||||
end
|
||||
|
||||
def sed_replace(regex)
|
||||
"sed 's/#{regex}/\\1/g'"
|
||||
end
|
||||
|
||||
def json_for_model model_name
|
||||
"mongoexport -d #{db_name} -c #{model_name} | #{id_sed} | #{date_sed}"
|
||||
end
|
||||
|
||||
def write_json_export
|
||||
log "Starting JSON export..."
|
||||
models.each do |model|
|
||||
log "Starting #{model[:name]} JSON export..."
|
||||
filename ="#{dirpath}/json/#{model[:name]}.json"
|
||||
model[:json_file] = filename
|
||||
`#{json_for_model(model[:name])} > #{filename}`
|
||||
debug "#{model[:name]} exported to #{dirname}/json/#{model[:name]}.json"
|
||||
log "Completed #{model[:name]} JSON export to #{dirname}/json/#{model[:name]}.json."
|
||||
end
|
||||
debug "Json export complete."
|
||||
log "JSON export complete."
|
||||
end
|
||||
def debug string
|
||||
|
||||
def log string
|
||||
if ['development', 'production'].include?(Rails.env)
|
||||
puts string
|
||||
puts "#{sprintf("%.2f", Time.now - start_time)}s #{string}"
|
||||
end
|
||||
Rails.logger.debug(string) if Rails.logger
|
||||
end
|
||||
|
||||
def convert_json_files
|
||||
models.each do |model|
|
||||
self.send("#{model[:name]}_json_to_csv".to_sym, model)
|
||||
end
|
||||
end
|
||||
|
||||
def generic_json_to_csv model_hash
|
||||
debug "Converting #{model_hash[:name]} json to csv"
|
||||
log "Converting #{model_hash[:name]} json to csv"
|
||||
json_file = File.open(model_hash[:json_file])
|
||||
|
||||
csv = CSV.open("#{dirpath}/csv/#{model_hash[:name]}.csv",'w')
|
||||
|
|
@ -89,6 +112,7 @@ class MongoToMysql
|
|||
json_file.close
|
||||
csv.close
|
||||
end
|
||||
|
||||
def comments_json_to_csv model_hash
|
||||
model_hash[:attrs] = ["mongo_id", "post_mongo_id", "person_mongo_id", "diaspora_handle", "text", "youtube_titles"]
|
||||
generic_json_to_csv(model_hash) do |hash|
|
||||
|
|
@ -96,6 +120,7 @@ class MongoToMysql
|
|||
mongo_attrs.map{|attr_name| hash[attr_name]}
|
||||
end
|
||||
end
|
||||
|
||||
def contacts_json_to_csv model_hash
|
||||
model_hash[:main_attrs] = ["mongo_id", "user_mongo_id", "person_mongo_id", "pending", "created_at", "updated_at"]
|
||||
#Post Visibilities
|
||||
|
|
@ -110,6 +135,7 @@ class MongoToMysql
|
|||
end
|
||||
#Also writes the aspect memberships csv
|
||||
end
|
||||
|
||||
def invitations_json_to_csv model_hash
|
||||
model_hash[:attrs] = ["mongo_id", "recipient_mongo_id", "sender_mongo_id", "aspect_mongo_id", "message"]
|
||||
generic_json_to_csv(model_hash) do |hash|
|
||||
|
|
@ -117,6 +143,7 @@ class MongoToMysql
|
|||
mongo_attrs.map{|attr_name| hash[attr_name]}
|
||||
end
|
||||
end
|
||||
|
||||
def notifications_json_to_csv model_hash
|
||||
model_hash[:attrs] = ["mongo_id", "target_id", "target_type", "unread"]
|
||||
generic_json_to_csv(model_hash) do |hash|
|
||||
|
|
@ -124,12 +151,13 @@ class MongoToMysql
|
|||
mongo_attrs.map{|attr_name| hash[attr_name]}
|
||||
end
|
||||
end
|
||||
|
||||
def people_json_to_csv model_hash
|
||||
model_hash[:attrs] = ["created_at", "updated_at", "serialized_public_key", "url", "mongo_id", "owner_mongo_id", "diaspora_handle"]
|
||||
model_hash[:profile_attrs] = ["image_url_medium", "searchable", "image_url", "person_mongo_id", "gender", "diaspora_handle", "birthday", "last_name", "bio", "image_url_small", "first_name"]
|
||||
#Also writes the profiles csv
|
||||
|
||||
debug "Converting #{model_hash[:name]} json to csv"
|
||||
log "Converting #{model_hash[:name]} json to csv"
|
||||
json_file = File.open(model_hash[:json_file])
|
||||
|
||||
people_csv = CSV.open("#{dirpath}/csv/#{model_hash[:name]}.csv",'w')
|
||||
|
|
@ -156,6 +184,7 @@ class MongoToMysql
|
|||
people_csv.close
|
||||
profiles_csv.close
|
||||
end
|
||||
|
||||
def posts_json_to_csv model_hash
|
||||
model_hash[:attrs] =["youtube_titles", "pending", "created_at", "public", "updated_at", "status_message_mongo_id", "caption", "remote_photo_path", "random_string", "image", "mongo_id", "type", "diaspora_handle", "person_mongo_id", "message" ]
|
||||
generic_json_to_csv(model_hash) do |hash|
|
||||
|
|
@ -164,6 +193,7 @@ class MongoToMysql
|
|||
end
|
||||
#has to handle the polymorphic stuff
|
||||
end
|
||||
|
||||
def requests_json_to_csv model_hash
|
||||
model_hash[:attrs] = ["mongo_id", "recipient_mongo_id", "sender_mongo_id", "aspect_mongo_id"]
|
||||
generic_json_to_csv(model_hash) do |hash|
|
||||
|
|
@ -171,6 +201,7 @@ class MongoToMysql
|
|||
mongo_attrs.map{|attr_name| hash[attr_name]}
|
||||
end
|
||||
end
|
||||
|
||||
def users_json_to_csv model_hash
|
||||
model_hash[:attrs] = ["mongo_id", "username", "serialized_private_key", "encrypted_password", "invites", "invitation_token", "invitation_sent_at", "getting_started", "disable_mail", "language", "last_sign_in_ip", "last_sign_in_at", "reset_password_token", "password_salt"]
|
||||
generic_json_to_csv(model_hash) do |hash|
|
||||
|
|
@ -178,8 +209,9 @@ class MongoToMysql
|
|||
mongo_attrs.map{|attr_name| hash[attr_name]}
|
||||
end
|
||||
end
|
||||
|
||||
def aspects_json_to_csv model_hash
|
||||
debug "Converting aspects json to aspects and post_visibilities csvs"
|
||||
log "Converting aspects json to aspects and post_visibilities csvs"
|
||||
model_hash[:main_attrs] = ["mongo_id", "name", "created_at", "updated_at"]
|
||||
#Post Visibilities
|
||||
model_hash[:join_table_name] = :post_visibilities
|
||||
|
|
@ -192,8 +224,9 @@ class MongoToMysql
|
|||
[main_row, post_visibility_rows]
|
||||
end
|
||||
end
|
||||
|
||||
def generic_json_to_two_csvs model_hash
|
||||
debug "Converting #{model_hash[:name]} json to two csvs"
|
||||
log "Converting #{model_hash[:name]} json to two csvs"
|
||||
json_file = File.open(model_hash[:json_file])
|
||||
|
||||
main_csv = CSV.open("#{dirpath}/csv/#{model_hash[:name]}.csv",'w')
|
||||
|
|
|
|||
|
|
@ -2,16 +2,16 @@
|
|||
# licensed under the Affero General Public License version 3 or later. See
|
||||
# the COPYRIGHT file.
|
||||
|
||||
require File.join(Rails.root, 'lib/rake_helpers')
|
||||
include RakeHelpers
|
||||
|
||||
namespace :migrations do
|
||||
desc 'export data for mysql import'
|
||||
task :export_for_mysql do
|
||||
require 'lib/mongo_to_mysql'
|
||||
require File.join(Rails.root, 'lib', 'mongo_to_mysql')
|
||||
migrator = MongoToMysql.new
|
||||
migrator.log("**** Starting export for MySQL ****")
|
||||
migrator.clear_dir
|
||||
migrator.write_json_export
|
||||
migrator.convert_json_files
|
||||
migrator.log("**** Export finished! ****")
|
||||
migrator.log("total elapsed time")
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Reference in a new issue