From e46b3e6cf3cad560b9dff534bbb47a49929dce40 Mon Sep 17 00:00:00 2001 From: Raphael Date: Thu, 6 Jan 2011 16:23:18 -0800 Subject: [PATCH] Import raw services, I think we need to change the name of the type field (is it supposed to be in provider?) other than that, only posts remain. --- app/models/mongo.rb | 2 +- .../20110105051803_create_import_tables.rb | 12 ++++++++++ db/schema.rb | 14 ++++++++++++ lib/data_conversion/base.rb | 12 +++++----- lib/data_conversion/export_from_mongo.rb | 6 ++++- lib/data_conversion/import_to_mysql.rb | 10 +++++++++ spec/fixtures/data_conversion/services.csv | 3 +++ .../data_conversion/import_to_mysql_spec.rb | 22 +++++++++++++++++++ 8 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 spec/fixtures/data_conversion/services.csv diff --git a/app/models/mongo.rb b/app/models/mongo.rb index 5b5b9cc74..e29d02fe7 100644 --- a/app/models/mongo.rb +++ b/app/models/mongo.rb @@ -15,7 +15,7 @@ module Mongo class PostVisibility < ActiveRecord::Base; end class Profile < ActiveRecord::Base; end class Request < ActiveRecord::Base; end - #Service? + class Service < ActiveRecord::Base; end #StatusMessage? class User < ActiveRecord::Base; end end diff --git a/db/migrate/20110105051803_create_import_tables.rb b/db/migrate/20110105051803_create_import_tables.rb index b2cf8a31e..25ff6cabf 100644 --- a/db/migrate/20110105051803_create_import_tables.rb +++ b/db/migrate/20110105051803_create_import_tables.rb @@ -112,6 +112,18 @@ class CreateImportTables < ActiveRecord::Migration add_index :mongo_requests, :recipient_mongo_id add_index :mongo_requests, [:sender_mongo_id, :recipient_mongo_id] + create_table :mongo_services do |t| + t.string :type + t.string :user_mongo_id + t.string :provider + t.string :uid + t.string :access_token + t.string :access_secret + t.string :nickname + t.timestamps + end + add_index :mongo_services, :user_mongo_id + create_table :mongo_users do |t| t.string :mongo_id t.string :username diff --git a/db/schema.rb b/db/schema.rb index 8a557a514..bbb2d73aa 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -203,6 +203,20 @@ ActiveRecord::Schema.define(:version => 20110105051803) do add_index "mongo_requests", ["sender_mongo_id", "recipient_mongo_id"], :name => "index_mongo_requests_on_sender_mongo_id_and_recipient_mongo_id" add_index "mongo_requests", ["sender_mongo_id"], :name => "index_mongo_requests_on_sender_mongo_id" + create_table "mongo_services", :force => true do |t| + t.string "type" + t.string "user_mongo_id" + t.string "provider" + t.string "uid" + t.string "access_token" + t.string "access_secret" + t.string "nickname" + t.datetime "created_at" + t.datetime "updated_at" + end + + add_index "mongo_services", ["user_mongo_id"], :name => "index_mongo_services_on_user_mongo_id" + create_table "mongo_users", :force => true do |t| t.string "mongo_id" t.string "username" diff --git a/lib/data_conversion/base.rb b/lib/data_conversion/base.rb index 6823e56cf..fd69907ee 100644 --- a/lib/data_conversion/base.rb +++ b/lib/data_conversion/base.rb @@ -37,16 +37,16 @@ module DataConversion :profile_attrs => ["image_url_medium", "searchable", "image_url", "person_mongo_id", "gender", "diaspora_handle", "birthday", "last_name", "bio", "image_url_small", "first_name"]}, {:name => :posts, :attrs => ["youtube_titles", "pending", "created_at", "public", "updated_at", "status_message_mongo_id", "caption", "remote_photo_path", "random_string", "image", "mongo_id", "type", "diaspora_handle", "person_mongo_id", "message"], - :mongo_attrs => ["youtube_titles", "pending", "created_at", "public", "updated_at", "status_message_id" , "caption", "remote_photo_path", "random_string", "image", "_id" , "_type", "diaspora_handle", "person_id" , "message"] - }, + :mongo_attrs => ["youtube_titles", "pending", "created_at", "public", "updated_at", "status_message_id" , "caption", "remote_photo_path", "random_string", "image", "_id" , "_type", "diaspora_handle", "person_id" , "message"]}, {:name => :requests, :attrs => ["mongo_id", "recipient_mongo_id", "sender_mongo_id", "aspect_mongo_id"], - :mongo_attrs => ["_id" , "to_id" , "from_id" , "into_id"] - }, + :mongo_attrs => ["_id" , "to_id" , "from_id" , "into_id"]}, + {:name => :services, + :attrs => ["type", "user_mongo_id", "provider", "uid", "access_token", "access_secret", "nickname"], + :mongo_attrs => ["_type", "user_id", "provider", "uid", "access_token", "access_secret", "nickname"]}, {:name => :users, :attrs => ["mongo_id", "username", "serialized_private_key", "encrypted_password", "invites", "invitation_token", "invitation_sent_at", "getting_started", "disable_mail", "language", "last_sign_in_ip", "last_sign_in_at", "reset_password_token", "password_salt"], - :mongo_attrs => ["_id", "username", "serialized_private_key", "encrypted_password", "invites", "invitation_token", "invitation_sent_at", "getting_started", "disable_mail", "language", "last_sign_in_ip", "last_sign_in_at", "reset_password_token", "password_salt"] - }, + :mongo_attrs => ["_id", "username", "serialized_private_key", "encrypted_password", "invites", "invitation_token", "invitation_sent_at", "getting_started", "disable_mail", "language", "last_sign_in_ip", "last_sign_in_at", "reset_password_token", "password_salt"]}, ] end def log(message) diff --git a/lib/data_conversion/export_from_mongo.rb b/lib/data_conversion/export_from_mongo.rb index d7ef4fdb8..6c2078b94 100644 --- a/lib/data_conversion/export_from_mongo.rb +++ b/lib/data_conversion/export_from_mongo.rb @@ -109,7 +109,11 @@ module DataConversion model_hash[:mongo_attrs].map { |attr_name| hash[attr_name] } end end - + def services_json_to_csv model_hash + generic_json_to_csv(model_hash) do |hash| + model_hash[:mongo_attrs].map { |attr_name| hash[attr_name] } + end + end def people_json_to_csv model_hash #Also writes the profiles csv diff --git a/lib/data_conversion/import_to_mysql.rb b/lib/data_conversion/import_to_mysql.rb index df5f5fb7c..5223baead 100644 --- a/lib/data_conversion/import_to_mysql.rb +++ b/lib/data_conversion/import_to_mysql.rb @@ -92,6 +92,16 @@ module DataConversion log "Finished. Imported #{Mongo::Contact.count} contacts." end + def import_raw_services + log "Loading services file..." + Mongo::Service.connection.execute <<-SQL + #{load_string("services")} + #{infile_opts} + (type,user_mongo_id,provider,uid,access_token,access_secret,nickname) + SQL + log "Finished. Imported #{Mongo::Service.count} services." + end + def import_raw_post_visibilities log "Loading post visibilities file..." Mongo::PostVisibility.connection.execute <<-SQL diff --git a/spec/fixtures/data_conversion/services.csv b/spec/fixtures/data_conversion/services.csv new file mode 100644 index 000000000..071403979 --- /dev/null +++ b/spec/fixtures/data_conversion/services.csv @@ -0,0 +1,3 @@ +type,user_mongo_id,provider,uid,access_token,access_secret,nickname +Services::Facebook,4d2657eacc8cb46033000011,,,yeah,, +Services::Twitter,4d2657e9cc8cb46033000009,,,yeah,foobar, diff --git a/spec/lib/data_conversion/import_to_mysql_spec.rb b/spec/lib/data_conversion/import_to_mysql_spec.rb index f559abd1c..4ea665a03 100644 --- a/spec/lib/data_conversion/import_to_mysql_spec.rb +++ b/spec/lib/data_conversion/import_to_mysql_spec.rb @@ -233,7 +233,29 @@ describe DataConversion::ImportToMysql do request.aspect_mongo_id.should == '' end end + describe "services" do + before do + copy_fixture_for("services") + end + it "imports data into the mongo_services table" do + Mongo::Service.count.should == 0 + @migrator.import_raw_services + Mongo::Service.count.should == 2 + end + + it "imports all the columns" do + @migrator.import_raw_services + service = Mongo::Service.first + service.type.should == "Services::Facebook" + service.user_mongo_id.should == "4d2657eacc8cb46033000011" + service.provider.should be_nil + service.uid.should be_nil + service.access_token.should == "yeah" + service.access_secret.should be_nil + service.nickname.should be_nil + end + end describe "users" do before do copy_fixture_for("users")