nocebo/diaspora
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactor ImportToMysql; add import_raw. Add rake task for import.

Browse source 
Remove unique constraint on index on mongo_requests.
This commit is contained in:
Sarah Mei 2011-01-05 23:54:22 -08:00
parent ff1cad6e63
commit 75ac7e91c8
4 changed files with 75 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
db/migrate/20110105051803_create_import_tables.rb
View file

@ -57,7 +57,7 @@ class CreateImportTables < ActiveRecord::Migration
end
add_index :mongo_requests, :sender_mongo_id
add_index :mongo_requests, :recipient_mongo_id
add_index :mongo_requests, [:sender_mongo_id, :recipient_mongo_id], :unique => true
add_index :mongo_requests, [:sender_mongo_id, :recipient_mongo_id]
create_table :mongo_users do |t|
t.string :mongo_id
@ -79,5 +79,11 @@ class CreateImportTables < ActiveRecord::Migration
def self.down
drop_table :mongo_users
drop_table :mongo_requests
drop_table :mongo_post_visibilities
drop_table :mongo_contacts
drop_table :mongo_comments
drop_table :mongo_aspect_memberships
drop_table :mongo_aspects
end
end

2
db/schema.rb
View file

@ -138,7 +138,7 @@ ActiveRecord::Schema.define(:version => 20110105051803) do
end
add_index "mongo_requests", ["recipient_mongo_id"], :name => "index_mongo_requests_on_recipient_mongo_id"
add_index "mongo_requests", ["sender_mongo_id", "recipient_mongo_id"], :name => "index_mongo_requests_on_sender_mongo_id_and_recipient_mongo_id", :unique => true
add_index "mongo_requests", ["sender_mongo_id", "recipient_mongo_id"], :name => "index_mongo_requests_on_sender_mongo_id_and_recipient_mongo_id"
add_index "mongo_requests", ["sender_mongo_id"], :name => "index_mongo_requests_on_sender_mongo_id"
create_table "mongo_users", :force => true do |t|

64
lib/data_conversion/import_to_mysql.rb
View file

@ -4,17 +4,34 @@
module DataConversion
class ImportToMysql < DataConversion::Base
def infile_opts
<<-OPTS
FIELDS TERMINATED BY ','
ENCLOSED BY '"'
IGNORE 1 LINES
OPTS
def import_raw
truncate_tables
import_raw_users
import_raw_aspects
import_raw_aspect_memberships
import_raw_comments
import_raw_contacts
import_raw_post_visibilities
import_raw_requests
end
def load_string model_name
"LOAD DATA INFILE '#{full_path}/#{model_name}.csv' INTO TABLE mongo_#{model_name}"
def process_raw_tables
end
def truncate_tables
Mongo::User.connection.execute "TRUNCATE TABLE mongo_users"
Mongo::Aspect.connection.execute "TRUNCATE TABLE mongo_aspects"
Mongo::AspectMembership.connection.execute "TRUNCATE TABLE mongo_aspect_memberships"
Mongo::Comment.connection.execute "TRUNCATE TABLE mongo_comments"
Mongo::Contact.connection.execute "TRUNCATE TABLE mongo_contacts"
Mongo::PostVisibility.connection.execute "TRUNCATE TABLE mongo_post_visibilities"
Mongo::Request.connection.execute "TRUNCATE TABLE mongo_requests"
end
def import_raw_users
log "Loading users file..."
Mongo::User.connection.execute <<-SQL
#{load_string("users")}
#{infile_opts}
@ -24,51 +41,80 @@ OPTS
reset_password_token, password_salt)
SET last_sign_in_at = FROM_UNIXTIME(LEFT(@last_sign_in_at_var, LENGTH(@last_sign_in_at_var)-3));
SQL
log "Finished. Imported #{Mongo::User.count} users."
end
def import_raw_aspects
log "Loading aspects file..."
Mongo::Aspect.connection.execute <<-SQL
#{load_string("aspects")}
#{infile_opts}
(mongo_id, name, user_mongo_id, @created_at, @updated_at)
SQL
log "Finished. Imported #{Mongo::Aspect.count} aspects."
end
def import_raw_aspect_memberships
log "Loading aspect memberships file..."
Mongo::Aspect.connection.execute <<-SQL
#{load_string("aspect_memberships")}
#{infile_opts}
(contact_mongo_id, aspect_mongo_id)
SQL
log "Finished. Imported #{Mongo::AspectMembership.count} aspect memberships."
end
def import_raw_comments
log "Loading comments file..."
Mongo::Aspect.connection.execute <<-SQL
#{load_string("comments")}
#{infile_opts}
(mongo_id, post_mongo_id, person_mongo_id, @diaspora_handle, text, youtube_titles)
SET guid = mongo_id;
SQL
log "Finished. Imported #{Mongo::Comment.count} comments."
end
def import_raw_contacts
log "Loading contacts file..."
Mongo::Aspect.connection.execute <<-SQL
#{load_string("contacts")}
#{infile_opts}
(mongo_id, user_mongo_id, person_mongo_id, pending, created_at, updated_at)
SQL
log "Finished. Imported #{Mongo::Contact.count} contacts."
end
def import_raw_post_visibilities
log "Loading post visibilities file..."
Mongo::Aspect.connection.execute <<-SQL
#{load_string("post_visibilities")}
#{infile_opts}
(aspect_mongo_id, post_mongo_id)
SQL
log "Finished. Imported #{Mongo::PostVisibility.count} post visibilities."
end
def import_raw_requests
log "Loading requests file..."
Mongo::Aspect.connection.execute <<-SQL
#{load_string("requests")}
#{infile_opts}
(mongo_id, recipient_mongo_id, sender_mongo_id, aspect_mongo_id)
SQL
log "Finished. Imported #{Mongo::Request.count} requests."
end
end
def infile_opts
<<-OPTS
FIELDS TERMINATED BY ','
ENCLOSED BY '"'
IGNORE 1 LINES
OPTS
end
def load_string model_name
"LOAD DATA INFILE '#{full_path}/#{model_name}.csv' INTO TABLE mongo_#{model_name}"
end
end
end

12
lib/tasks/migrations.rake
View file

@ -8,6 +8,7 @@ namespace :migrations do
desc 'export data for mysql import'
task :export_for_mysql do
migrator = DataConversion::ExportFromMongo.new
migrator.full_path = "/tmp/data_conversion"
migrator.log("**** Starting export for MySQL ****")
migrator.clear_dir
migrator.write_json_export
@ -15,4 +16,15 @@ namespace :migrations do
migrator.log("**** Export finished! ****")
migrator.log("total elapsed time")
end
desc 'import data to mysql'
task :import_to_mysql => :environment do
migrator = DataConversion::ImportToMysql.new
migrator.full_path = "/tmp/data_conversion/csv"
migrator.log("**** Starting import to MySQL database #{ActiveRecord::Base.connection.current_database} ****")
migrator.import_raw
migrator.process_raw_tables
migrator.log("**** Import finished! ****")
migrator.log("total elapsed time")
end
end