From 75ac7e91c859cf65520d2ffb9cf8209c1e2e5b07 Mon Sep 17 00:00:00 2001 From: Sarah Mei Date: Wed, 5 Jan 2011 23:54:22 -0800 Subject: [PATCH] Refactor ImportToMysql; add import_raw. Add rake task for import. Remove unique constraint on index on mongo_requests. --- .../20110105051803_create_import_tables.rb | 8 ++- db/schema.rb | 2 +- lib/data_conversion/import_to_mysql.rb | 64 ++++++++++++++++--- lib/tasks/migrations.rake | 12 ++++ 4 files changed, 75 insertions(+), 11 deletions(-) diff --git a/db/migrate/20110105051803_create_import_tables.rb b/db/migrate/20110105051803_create_import_tables.rb index cc38317de..452569602 100644 --- a/db/migrate/20110105051803_create_import_tables.rb +++ b/db/migrate/20110105051803_create_import_tables.rb @@ -57,7 +57,7 @@ class CreateImportTables < ActiveRecord::Migration end add_index :mongo_requests, :sender_mongo_id add_index :mongo_requests, :recipient_mongo_id - add_index :mongo_requests, [:sender_mongo_id, :recipient_mongo_id], :unique => true + add_index :mongo_requests, [:sender_mongo_id, :recipient_mongo_id] create_table :mongo_users do |t| t.string :mongo_id @@ -79,5 +79,11 @@ class CreateImportTables < ActiveRecord::Migration def self.down drop_table :mongo_users + drop_table :mongo_requests + drop_table :mongo_post_visibilities + drop_table :mongo_contacts + drop_table :mongo_comments + drop_table :mongo_aspect_memberships + drop_table :mongo_aspects end end diff --git a/db/schema.rb b/db/schema.rb index 9dab91981..c833b65ec 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -138,7 +138,7 @@ ActiveRecord::Schema.define(:version => 20110105051803) do end add_index "mongo_requests", ["recipient_mongo_id"], :name => "index_mongo_requests_on_recipient_mongo_id" - add_index "mongo_requests", ["sender_mongo_id", "recipient_mongo_id"], :name => "index_mongo_requests_on_sender_mongo_id_and_recipient_mongo_id", :unique => true + add_index "mongo_requests", ["sender_mongo_id", "recipient_mongo_id"], :name => "index_mongo_requests_on_sender_mongo_id_and_recipient_mongo_id" add_index "mongo_requests", ["sender_mongo_id"], :name => "index_mongo_requests_on_sender_mongo_id" create_table "mongo_users", :force => true do |t| diff --git a/lib/data_conversion/import_to_mysql.rb b/lib/data_conversion/import_to_mysql.rb index 70fc77dd7..e4493cf3a 100644 --- a/lib/data_conversion/import_to_mysql.rb +++ b/lib/data_conversion/import_to_mysql.rb @@ -4,17 +4,34 @@ module DataConversion class ImportToMysql < DataConversion::Base - def infile_opts - <<-OPTS - FIELDS TERMINATED BY ',' - ENCLOSED BY '"' - IGNORE 1 LINES -OPTS + + def import_raw + truncate_tables + import_raw_users + import_raw_aspects + import_raw_aspect_memberships + import_raw_comments + import_raw_contacts + import_raw_post_visibilities + import_raw_requests end - def load_string model_name - "LOAD DATA INFILE '#{full_path}/#{model_name}.csv' INTO TABLE mongo_#{model_name}" + + def process_raw_tables + end + + def truncate_tables + Mongo::User.connection.execute "TRUNCATE TABLE mongo_users" + Mongo::Aspect.connection.execute "TRUNCATE TABLE mongo_aspects" + Mongo::AspectMembership.connection.execute "TRUNCATE TABLE mongo_aspect_memberships" + Mongo::Comment.connection.execute "TRUNCATE TABLE mongo_comments" + Mongo::Contact.connection.execute "TRUNCATE TABLE mongo_contacts" + Mongo::PostVisibility.connection.execute "TRUNCATE TABLE mongo_post_visibilities" + Mongo::Request.connection.execute "TRUNCATE TABLE mongo_requests" + end + def import_raw_users + log "Loading users file..." Mongo::User.connection.execute <<-SQL #{load_string("users")} #{infile_opts} @@ -24,51 +41,80 @@ OPTS reset_password_token, password_salt) SET last_sign_in_at = FROM_UNIXTIME(LEFT(@last_sign_in_at_var, LENGTH(@last_sign_in_at_var)-3)); SQL + log "Finished. Imported #{Mongo::User.count} users." end + def import_raw_aspects + log "Loading aspects file..." Mongo::Aspect.connection.execute <<-SQL #{load_string("aspects")} #{infile_opts} (mongo_id, name, user_mongo_id, @created_at, @updated_at) SQL + log "Finished. Imported #{Mongo::Aspect.count} aspects." end + def import_raw_aspect_memberships + log "Loading aspect memberships file..." Mongo::Aspect.connection.execute <<-SQL #{load_string("aspect_memberships")} #{infile_opts} (contact_mongo_id, aspect_mongo_id) SQL + log "Finished. Imported #{Mongo::AspectMembership.count} aspect memberships." end + def import_raw_comments + log "Loading comments file..." Mongo::Aspect.connection.execute <<-SQL #{load_string("comments")} #{infile_opts} (mongo_id, post_mongo_id, person_mongo_id, @diaspora_handle, text, youtube_titles) SET guid = mongo_id; SQL + log "Finished. Imported #{Mongo::Comment.count} comments." end + def import_raw_contacts + log "Loading contacts file..." Mongo::Aspect.connection.execute <<-SQL #{load_string("contacts")} #{infile_opts} (mongo_id, user_mongo_id, person_mongo_id, pending, created_at, updated_at) SQL + log "Finished. Imported #{Mongo::Contact.count} contacts." end + def import_raw_post_visibilities + log "Loading post visibilities file..." Mongo::Aspect.connection.execute <<-SQL #{load_string("post_visibilities")} #{infile_opts} (aspect_mongo_id, post_mongo_id) SQL + log "Finished. Imported #{Mongo::PostVisibility.count} post visibilities." end + def import_raw_requests + log "Loading requests file..." Mongo::Aspect.connection.execute <<-SQL #{load_string("requests")} #{infile_opts} (mongo_id, recipient_mongo_id, sender_mongo_id, aspect_mongo_id) SQL + log "Finished. Imported #{Mongo::Request.count} requests." end - end + def infile_opts + <<-OPTS + FIELDS TERMINATED BY ',' + ENCLOSED BY '"' + IGNORE 1 LINES +OPTS + end + def load_string model_name + "LOAD DATA INFILE '#{full_path}/#{model_name}.csv' INTO TABLE mongo_#{model_name}" + end + end end diff --git a/lib/tasks/migrations.rake b/lib/tasks/migrations.rake index 506138c7e..b4a8d0283 100644 --- a/lib/tasks/migrations.rake +++ b/lib/tasks/migrations.rake @@ -8,6 +8,7 @@ namespace :migrations do desc 'export data for mysql import' task :export_for_mysql do migrator = DataConversion::ExportFromMongo.new + migrator.full_path = "/tmp/data_conversion" migrator.log("**** Starting export for MySQL ****") migrator.clear_dir migrator.write_json_export @@ -15,4 +16,15 @@ namespace :migrations do migrator.log("**** Export finished! ****") migrator.log("total elapsed time") end + + desc 'import data to mysql' + task :import_to_mysql => :environment do + migrator = DataConversion::ImportToMysql.new + migrator.full_path = "/tmp/data_conversion/csv" + migrator.log("**** Starting import to MySQL database #{ActiveRecord::Base.connection.current_database} ****") + migrator.import_raw + migrator.process_raw_tables + migrator.log("**** Import finished! ****") + migrator.log("total elapsed time") + end end