From 283a5501bef009e6227fdbd5bf4529e11452e512 Mon Sep 17 00:00:00 2001 From: Raphael Date: Thu, 6 Jan 2011 14:58:42 -0800 Subject: [PATCH] Boolean values aren't importing; we may need SET statements for them --- app/models/mongo.rb | 2 +- .../20110105051803_create_import_tables.rb | 20 ++++++ db/schema.rb | 21 ++++++ lib/data_conversion/export_from_mongo.rb | 5 +- lib/data_conversion/import_to_mysql.rb | 10 ++- spec/fixtures/data_conversion/profiles.csv | 70 +++---------------- .../data_conversion/import_to_mysql_spec.rb | 29 +++++++- 7 files changed, 91 insertions(+), 66 deletions(-) diff --git a/app/models/mongo.rb b/app/models/mongo.rb index 8f790f0cb..5b5b9cc74 100644 --- a/app/models/mongo.rb +++ b/app/models/mongo.rb @@ -13,7 +13,7 @@ module Mongo #Photo? #Post? class PostVisibility < ActiveRecord::Base; end - #class Profile < ActiveRecord::Base; end + class Profile < ActiveRecord::Base; end class Request < ActiveRecord::Base; end #Service? #StatusMessage? diff --git a/db/migrate/20110105051803_create_import_tables.rb b/db/migrate/20110105051803_create_import_tables.rb index fe572d003..b2cf8a31e 100644 --- a/db/migrate/20110105051803_create_import_tables.rb +++ b/db/migrate/20110105051803_create_import_tables.rb @@ -81,6 +81,26 @@ class CreateImportTables < ActiveRecord::Migration add_index :mongo_post_visibilities, :aspect_mongo_id add_index :mongo_post_visibilities, :post_mongo_id + create_table :mongo_profiles do |t| + t.string :diaspora_handle + t.string :first_name + t.string :last_name + t.string :image_url + t.string :image_url_small + t.string :image_url_medium + t.date :birthday + t.string :gender + t.text :bio + t.boolean :searchable, :default => true + t.string :person_mongo_id + t.timestamps + end + add_index :mongo_profiles, [:first_name, :searchable] + add_index :mongo_profiles, [:last_name, :searchable] + add_index :mongo_profiles, [:first_name, :last_name, :searchable] + add_index :mongo_profiles, :person_mongo_id + + create_table :mongo_requests do |t| t.string :mongo_id t.string :sender_mongo_id diff --git a/db/schema.rb b/db/schema.rb index 76b10bab5..8a557a514 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -169,6 +169,27 @@ ActiveRecord::Schema.define(:version => 20110105051803) do add_index "mongo_post_visibilities", ["aspect_mongo_id"], :name => "index_mongo_post_visibilities_on_aspect_mongo_id" add_index "mongo_post_visibilities", ["post_mongo_id"], :name => "index_mongo_post_visibilities_on_post_mongo_id" + create_table "mongo_profiles", :force => true do |t| + t.string "diaspora_handle" + t.string "first_name" + t.string "last_name" + t.string "image_url" + t.string "image_url_small" + t.string "image_url_medium" + t.date "birthday" + t.string "gender" + t.text "bio" + t.boolean "searchable", :default => true + t.string "person_mongo_id" + t.datetime "created_at" + t.datetime "updated_at" + end + + add_index "mongo_profiles", ["first_name", "last_name", "searchable"], :name => "index_mongo_profiles_on_first_name_and_last_name_and_searchable" + add_index "mongo_profiles", ["first_name", "searchable"], :name => "index_mongo_profiles_on_first_name_and_searchable" + add_index "mongo_profiles", ["last_name", "searchable"], :name => "index_mongo_profiles_on_last_name_and_searchable" + add_index "mongo_profiles", ["person_mongo_id"], :name => "index_mongo_profiles_on_person_mongo_id" + create_table "mongo_requests", :force => true do |t| t.string "mongo_id" t.string "sender_mongo_id" diff --git a/lib/data_conversion/export_from_mongo.rb b/lib/data_conversion/export_from_mongo.rb index 68756a903..d7ef4fdb8 100644 --- a/lib/data_conversion/export_from_mongo.rb +++ b/lib/data_conversion/export_from_mongo.rb @@ -132,11 +132,10 @@ module DataConversion people_csv << person_row profile_row = model_hash[:profile_attrs].map do |attr_name| - attr_name = attr_name.gsub("mongo_", "") - attr_name = "_id" if attr_name == "id" + attr_name = "_id" if attr_name == "person_mongo_id" hash["profile"][attr_name] end - profiles_csv << person_row + profiles_csv << profile_row end json_file.close people_csv.close diff --git a/lib/data_conversion/import_to_mysql.rb b/lib/data_conversion/import_to_mysql.rb index 16ee6b178..cfaead596 100644 --- a/lib/data_conversion/import_to_mysql.rb +++ b/lib/data_conversion/import_to_mysql.rb @@ -132,7 +132,15 @@ module DataConversion SQL log "Finished. Imported #{Mongo::Person.count} people." end - + def import_raw_profiles + log "Loading profiles file..." + Mongo::Profile.connection.execute <<-SQL + #{load_string("profiles")} + #{infile_opts} + (image_url_medium,searchable,image_url,person_mongo_id,gender,diaspora_handle,birthday,last_name,bio,image_url_small,first_name) + SQL + log "Finished. Imported #{Mongo::Profile.count} profiles." + end def infile_opts <<-OPTS FIELDS TERMINATED BY ',' diff --git a/spec/fixtures/data_conversion/profiles.csv b/spec/fixtures/data_conversion/profiles.csv index b28edd803..038ad46c6 100644 --- a/spec/fixtures/data_conversion/profiles.csv +++ b/spec/fixtures/data_conversion/profiles.csv @@ -1,61 +1,11 @@ image_url_medium,searchable,image_url,person_mongo_id,gender,diaspora_handle,birthday,last_name,bio,image_url_small,first_name -1294344490000,1294344490000,"-----BEGIN RSA PUBLIC KEY----- -MIGJAoGBAM6tu6ynfN7NTWXWpZNIeHgGCxjdNmDt65DE0Nhduvng7ZXbyciRVtUc -miFwCOVBEQNS+NLslSaGidcGc6ASXUUWl7sZiHTJ3y2ykKE09gGz0yYgsIaZVGEQ -tcCPLgNRsNL4p+JLkNxGTQcj1I0oAqJ1hPT5EN+XE9BcEULLBv4DAgMBAAE= ------END RSA PUBLIC KEY----- -",http://google-1b5b16a.com/,4d26212acc8cb44df2000002,,bob-person-1a8bc18@aol.com -1294344490000,1294344490000,"-----BEGIN RSA PUBLIC KEY----- -MIGJAoGBAM6tu6ynfN7NTWXWpZNIeHgGCxjdNmDt65DE0Nhduvng7ZXbyciRVtUc -miFwCOVBEQNS+NLslSaGidcGc6ASXUUWl7sZiHTJ3y2ykKE09gGz0yYgsIaZVGEQ -tcCPLgNRsNL4p+JLkNxGTQcj1I0oAqJ1hPT5EN+XE9BcEULLBv4DAgMBAAE= ------END RSA PUBLIC KEY----- -",http://google-2bf625d.com/,4d26212acc8cb44df2000003,,bob-person-2e06150@aol.com -1294344490000,1294344490000,"-----BEGIN RSA PUBLIC KEY----- -MIGJAoGBAM6tu6ynfN7NTWXWpZNIeHgGCxjdNmDt65DE0Nhduvng7ZXbyciRVtUc -miFwCOVBEQNS+NLslSaGidcGc6ASXUUWl7sZiHTJ3y2ykKE09gGz0yYgsIaZVGEQ -tcCPLgNRsNL4p+JLkNxGTQcj1I0oAqJ1hPT5EN+XE9BcEULLBv4DAgMBAAE= ------END RSA PUBLIC KEY----- -",http://google-316cae4.com/,4d26212acc8cb44df2000004,,bob-person-30d59c1@aol.com -1294344491000,1294344500000,"-----BEGIN RSA PUBLIC KEY----- -MIGJAoGBALvHUWZmDIJ5DJpjtacVH/oCox+UtMKKiq+2nUsQC8TD1kanmVyUYK8Y -E4sy1zI56E/wKmC/a4UMEQsapFX0CtRhyXEoikoeKUQCATWKjlQVHUmNYybqLDsu -WenFygxQn9ZpcdKxVnQ6kyd6rTd98d6xtphnM2N5eVk1gRKdvsSFAgMBAAE= ------END RSA PUBLIC KEY----- -",http://google-45085cf.com/,4d26212bcc8cb44df2000008,4d26212acc8cb44df2000005,bob178fa79@localhost -1294344491000,1294344499000,"-----BEGIN RSA PUBLIC KEY----- -MIGJAoGBALvHUWZmDIJ5DJpjtacVH/oCox+UtMKKiq+2nUsQC8TD1kanmVyUYK8Y -E4sy1zI56E/wKmC/a4UMEQsapFX0CtRhyXEoikoeKUQCATWKjlQVHUmNYybqLDsu -WenFygxQn9ZpcdKxVnQ6kyd6rTd98d6xtphnM2N5eVk1gRKdvsSFAgMBAAE= ------END RSA PUBLIC KEY----- -",http://google-5384520.com/,4d26212bcc8cb44df200000c,4d26212bcc8cb44df2000009,bob2f2d21a@localhost -1294344491000,1294344491000,"-----BEGIN RSA PUBLIC KEY----- -MIGJAoGBAM6tu6ynfN7NTWXWpZNIeHgGCxjdNmDt65DE0Nhduvng7ZXbyciRVtUc -miFwCOVBEQNS+NLslSaGidcGc6ASXUUWl7sZiHTJ3y2ykKE09gGz0yYgsIaZVGEQ -tcCPLgNRsNL4p+JLkNxGTQcj1I0oAqJ1hPT5EN+XE9BcEULLBv4DAgMBAAE= ------END RSA PUBLIC KEY----- -",http://google-672f1da.com/,4d26212bcc8cb44df200000f,,bob-person-46276e5@aol.com -1294344491000,1294344500000,"-----BEGIN RSA PUBLIC KEY----- -MIGJAoGBALvHUWZmDIJ5DJpjtacVH/oCox+UtMKKiq+2nUsQC8TD1kanmVyUYK8Y -E4sy1zI56E/wKmC/a4UMEQsapFX0CtRhyXEoikoeKUQCATWKjlQVHUmNYybqLDsu -WenFygxQn9ZpcdKxVnQ6kyd6rTd98d6xtphnM2N5eVk1gRKdvsSFAgMBAAE= ------END RSA PUBLIC KEY----- -",http://google-7c2ce04.com/,4d26212bcc8cb44df2000014,4d26212bcc8cb44df2000011,bob3127b24@localhost -1294344491000,1294344502000,"-----BEGIN RSA PUBLIC KEY----- -MIGJAoGBALvHUWZmDIJ5DJpjtacVH/oCox+UtMKKiq+2nUsQC8TD1kanmVyUYK8Y -E4sy1zI56E/wKmC/a4UMEQsapFX0CtRhyXEoikoeKUQCATWKjlQVHUmNYybqLDsu -WenFygxQn9ZpcdKxVnQ6kyd6rTd98d6xtphnM2N5eVk1gRKdvsSFAgMBAAE= ------END RSA PUBLIC KEY----- -",http://google-89e13a6.com/,4d26212bcc8cb44df2000018,4d26212bcc8cb44df2000015,bob4221dca@localhost -1294344492000,1294344492000,"-----BEGIN RSA PUBLIC KEY----- -MIGJAoGBAM6tu6ynfN7NTWXWpZNIeHgGCxjdNmDt65DE0Nhduvng7ZXbyciRVtUc -miFwCOVBEQNS+NLslSaGidcGc6ASXUUWl7sZiHTJ3y2ykKE09gGz0yYgsIaZVGEQ -tcCPLgNRsNL4p+JLkNxGTQcj1I0oAqJ1hPT5EN+XE9BcEULLBv4DAgMBAAE= ------END RSA PUBLIC KEY----- -",http://google-99bfbf0.com/,4d26212ccc8cb44df200001d,,bob-person-594e154@aol.com -1294344501000,1294344502000,"-----BEGIN RSA PUBLIC KEY----- -MIGJAoGBALvHUWZmDIJ5DJpjtacVH/oCox+UtMKKiq+2nUsQC8TD1kanmVyUYK8Y -E4sy1zI56E/wKmC/a4UMEQsapFX0CtRhyXEoikoeKUQCATWKjlQVHUmNYybqLDsu -WenFygxQn9ZpcdKxVnQ6kyd6rTd98d6xtphnM2N5eVk1gRKdvsSFAgMBAAE= ------END RSA PUBLIC KEY----- -",http://google-100d6ffc.com/,4d262135cc8cb44df2000034,,bob5ae60b9@localhost +,true,,4d262129cc8cb44df2000001,,,,weinstien,,,eugene +,true,,4d262129cc8cb44df2000001,,,,weinstien,,,eugene +,true,,4d262129cc8cb44df2000001,,,,weinstien,,,eugene +,true,,4d26212bcc8cb44df2000007,,,,Grimm172d9ad,,,Robert1c2ef4f +,true,,4d26212bcc8cb44df200000b,,,,Grimm2b840b9,,,Robert27f81ab +,true,,4d262129cc8cb44df2000001,,,,weinstien,,,eugene +,true,,4d26212bcc8cb44df2000013,,,,Grimm3402a1c,,,Robert3eae7ca +,true,,4d26212bcc8cb44df2000017,,,,Grimm4d8d32a,,,Robert49ae3ee +,true,,4d262129cc8cb44df2000001,,,,weinstien,,,eugene +,true,,4d262135cc8cb44df2000033,,,,Grimm58c83c9,,,Robert5d5a980 diff --git a/spec/lib/data_conversion/import_to_mysql_spec.rb b/spec/lib/data_conversion/import_to_mysql_spec.rb index eb2fde384..6301ad1d9 100644 --- a/spec/lib/data_conversion/import_to_mysql_spec.rb +++ b/spec/lib/data_conversion/import_to_mysql_spec.rb @@ -152,7 +152,7 @@ describe DataConversion::ImportToMysql do it "imports data into the mongo_people table" do Mongo::Person.count.should == 0 @migrator.import_raw_people - Mongo::Person.count.should == 6 + Mongo::Person.count.should == 10 end it "imports all the columns" do @@ -182,6 +182,33 @@ describe DataConversion::ImportToMysql do pv.aspect_mongo_id.should =="4d26212bcc8cb44df2000006" end end + describe "profiles" do + before do + copy_fixture_for("profiles") + end + + it "imports data into the mongo_profiles table" do + Mongo::Profile.count.should == 0 + @migrator.import_raw_profiles + Mongo::Profile.count.should == 10 + end + + it "imports all the columns" do + @migrator.import_raw_profiles + profile = Mongo::Profile.first + profile.image_url_medium.should == '' + profile.searchable.should == true + profile.image_url.should == '' + profile.person_mongo_id.should == "4d262129cc8cb44df2000001" + profile.gender.should == '' + profile.diaspora_handle.should == '' + profile.birthday.should == '' + profile.last_name.should == 'weinstein' + profile.bio.should == '' + profile.image_url_small.should == '' + profile.first_name.should == 'eugene' + end + end describe "requests" do before do