Refactor ImportToMysql; add import_raw. Add rake task for import.

Remove unique constraint on index on mongo_requests.

db/migrate/20110105051803_create_import_tables.rb

@@ -57,7 +57,7 @@ class CreateImportTables < ActiveRecord::Migration
     end
     add_index :mongo_requests, :sender_mongo_id
     add_index :mongo_requests, :recipient_mongo_id
-    add_index :mongo_requests, [:sender_mongo_id, :recipient_mongo_id], :unique => true
+    add_index :mongo_requests, [:sender_mongo_id, :recipient_mongo_id]
 
     create_table :mongo_users do |t|
       t.string :mongo_id
@@ -79,5 +79,11 @@ class CreateImportTables < ActiveRecord::Migration
 
   def self.down
     drop_table :mongo_users
+    drop_table :mongo_requests
+    drop_table :mongo_post_visibilities
+    drop_table :mongo_contacts
+    drop_table :mongo_comments
+    drop_table :mongo_aspect_memberships
+    drop_table :mongo_aspects
   end
 end

db/schema.rb

@@ -138,7 +138,7 @@ ActiveRecord::Schema.define(:version => 20110105051803) do
   end
 
   add_index "mongo_requests", ["recipient_mongo_id"], :name => "index_mongo_requests_on_recipient_mongo_id"
-  add_index "mongo_requests", ["sender_mongo_id", "recipient_mongo_id"], :name => "index_mongo_requests_on_sender_mongo_id_and_recipient_mongo_id", :unique => true
+  add_index "mongo_requests", ["sender_mongo_id", "recipient_mongo_id"], :name => "index_mongo_requests_on_sender_mongo_id_and_recipient_mongo_id"
   add_index "mongo_requests", ["sender_mongo_id"], :name => "index_mongo_requests_on_sender_mongo_id"
 
   create_table "mongo_users", :force => true do |t|

lib/data_conversion/import_to_mysql.rb

@@ -4,17 +4,34 @@
 
 module DataConversion
   class ImportToMysql < DataConversion::Base
-    def infile_opts
+
-      <<-OPTS
+    def import_raw
-          FIELDS TERMINATED BY ','
+      truncate_tables
-          ENCLOSED BY '"'
+      import_raw_users
-          IGNORE 1 LINES
+      import_raw_aspects
-OPTS
+      import_raw_aspect_memberships
+      import_raw_comments
+      import_raw_contacts
+      import_raw_post_visibilities
+      import_raw_requests
     end
-    def load_string model_name
+
-        "LOAD DATA INFILE '#{full_path}/#{model_name}.csv' INTO TABLE mongo_#{model_name}"
+    def process_raw_tables
+      
     end
+    
+    def truncate_tables
+      Mongo::User.connection.execute "TRUNCATE TABLE mongo_users"
+      Mongo::Aspect.connection.execute "TRUNCATE TABLE mongo_aspects"
+      Mongo::AspectMembership.connection.execute "TRUNCATE TABLE mongo_aspect_memberships"
+      Mongo::Comment.connection.execute "TRUNCATE TABLE mongo_comments"
+      Mongo::Contact.connection.execute "TRUNCATE TABLE mongo_contacts"
+      Mongo::PostVisibility.connection.execute "TRUNCATE TABLE mongo_post_visibilities"
+      Mongo::Request.connection.execute "TRUNCATE TABLE mongo_requests"
+    end
+
     def import_raw_users
+      log "Loading users file..."
       Mongo::User.connection.execute <<-SQL
         #{load_string("users")}
         #{infile_opts}
@@ -24,51 +41,80 @@ OPTS
          reset_password_token, password_salt)
          SET last_sign_in_at = FROM_UNIXTIME(LEFT(@last_sign_in_at_var, LENGTH(@last_sign_in_at_var)-3));
       SQL
+      log "Finished. Imported #{Mongo::User.count} users."
     end
+
     def import_raw_aspects
+      log "Loading aspects file..."
       Mongo::Aspect.connection.execute <<-SQL
         #{load_string("aspects")}
         #{infile_opts}
         (mongo_id, name, user_mongo_id, @created_at, @updated_at)
       SQL
+      log "Finished. Imported #{Mongo::Aspect.count} aspects."
     end
+
     def import_raw_aspect_memberships
+      log "Loading aspect memberships file..."
       Mongo::Aspect.connection.execute <<-SQL
         #{load_string("aspect_memberships")}
         #{infile_opts}
         (contact_mongo_id, aspect_mongo_id)
       SQL
+      log "Finished. Imported #{Mongo::AspectMembership.count} aspect memberships."
     end
+
     def import_raw_comments
+      log "Loading comments file..."
       Mongo::Aspect.connection.execute <<-SQL
         #{load_string("comments")}
         #{infile_opts}
         (mongo_id, post_mongo_id, person_mongo_id, @diaspora_handle, text, youtube_titles)
         SET guid = mongo_id;
       SQL
+      log "Finished. Imported #{Mongo::Comment.count} comments."
     end
+
     def import_raw_contacts
+      log "Loading contacts file..."
       Mongo::Aspect.connection.execute <<-SQL
         #{load_string("contacts")}
         #{infile_opts}
         (mongo_id, user_mongo_id, person_mongo_id, pending, created_at, updated_at)
       SQL
+      log "Finished. Imported #{Mongo::Contact.count} contacts."
     end
+
     def import_raw_post_visibilities
+      log "Loading post visibilities file..."
       Mongo::Aspect.connection.execute <<-SQL
         #{load_string("post_visibilities")}
         #{infile_opts}
         (aspect_mongo_id, post_mongo_id)
       SQL
+      log "Finished. Imported #{Mongo::PostVisibility.count} post visibilities."
     end
+
     def import_raw_requests
+      log "Loading requests file..."
       Mongo::Aspect.connection.execute <<-SQL
         #{load_string("requests")}
         #{infile_opts}
         (mongo_id, recipient_mongo_id, sender_mongo_id, aspect_mongo_id)
       SQL
+      log "Finished. Imported #{Mongo::Request.count} requests."
     end
 
-  end
+    def infile_opts
+      <<-OPTS
+          FIELDS TERMINATED BY ','
+          ENCLOSED BY '"'
+          IGNORE 1 LINES
+OPTS
+    end
 
+    def load_string model_name
+        "LOAD DATA INFILE '#{full_path}/#{model_name}.csv' INTO TABLE mongo_#{model_name}"
+    end
+  end
 end

lib/tasks/migrations.rake

@@ -8,6 +8,7 @@ namespace :migrations do
   desc 'export data for mysql import'
   task :export_for_mysql do
     migrator = DataConversion::ExportFromMongo.new
+    migrator.full_path = "/tmp/data_conversion"
     migrator.log("**** Starting export for MySQL ****")
     migrator.clear_dir
     migrator.write_json_export
@@ -15,4 +16,15 @@ namespace :migrations do
     migrator.log("**** Export finished! ****")
     migrator.log("total elapsed time")
   end
+
+  desc 'import data to mysql'
+  task :import_to_mysql => :environment do
+    migrator = DataConversion::ImportToMysql.new
+    migrator.full_path = "/tmp/data_conversion/csv"
+    migrator.log("**** Starting import to MySQL database #{ActiveRecord::Base.connection.current_database} ****")
+    migrator.import_raw
+    migrator.process_raw_tables
+    migrator.log("**** Import finished! ****")
+    migrator.log("total elapsed time")
+  end
 end