Import raw services, I think we need to change the name of the type field (is it supposed to be in provider?) other than that, only posts remain.

This commit is contained in:
Raphael 2011-01-06 16:23:18 -08:00
parent 7668276a99
commit e46b3e6cf3
8 changed files with 73 additions and 8 deletions

View file

@ -15,7 +15,7 @@ module Mongo
class PostVisibility < ActiveRecord::Base; end class PostVisibility < ActiveRecord::Base; end
class Profile < ActiveRecord::Base; end class Profile < ActiveRecord::Base; end
class Request < ActiveRecord::Base; end class Request < ActiveRecord::Base; end
#Service? class Service < ActiveRecord::Base; end
#StatusMessage? #StatusMessage?
class User < ActiveRecord::Base; end class User < ActiveRecord::Base; end
end end

View file

@ -112,6 +112,18 @@ class CreateImportTables < ActiveRecord::Migration
add_index :mongo_requests, :recipient_mongo_id add_index :mongo_requests, :recipient_mongo_id
add_index :mongo_requests, [:sender_mongo_id, :recipient_mongo_id] add_index :mongo_requests, [:sender_mongo_id, :recipient_mongo_id]
create_table :mongo_services do |t|
t.string :type
t.string :user_mongo_id
t.string :provider
t.string :uid
t.string :access_token
t.string :access_secret
t.string :nickname
t.timestamps
end
add_index :mongo_services, :user_mongo_id
create_table :mongo_users do |t| create_table :mongo_users do |t|
t.string :mongo_id t.string :mongo_id
t.string :username t.string :username

View file

@ -203,6 +203,20 @@ ActiveRecord::Schema.define(:version => 20110105051803) do
add_index "mongo_requests", ["sender_mongo_id", "recipient_mongo_id"], :name => "index_mongo_requests_on_sender_mongo_id_and_recipient_mongo_id" add_index "mongo_requests", ["sender_mongo_id", "recipient_mongo_id"], :name => "index_mongo_requests_on_sender_mongo_id_and_recipient_mongo_id"
add_index "mongo_requests", ["sender_mongo_id"], :name => "index_mongo_requests_on_sender_mongo_id" add_index "mongo_requests", ["sender_mongo_id"], :name => "index_mongo_requests_on_sender_mongo_id"
create_table "mongo_services", :force => true do |t|
t.string "type"
t.string "user_mongo_id"
t.string "provider"
t.string "uid"
t.string "access_token"
t.string "access_secret"
t.string "nickname"
t.datetime "created_at"
t.datetime "updated_at"
end
add_index "mongo_services", ["user_mongo_id"], :name => "index_mongo_services_on_user_mongo_id"
create_table "mongo_users", :force => true do |t| create_table "mongo_users", :force => true do |t|
t.string "mongo_id" t.string "mongo_id"
t.string "username" t.string "username"

View file

@ -37,16 +37,16 @@ module DataConversion
:profile_attrs => ["image_url_medium", "searchable", "image_url", "person_mongo_id", "gender", "diaspora_handle", "birthday", "last_name", "bio", "image_url_small", "first_name"]}, :profile_attrs => ["image_url_medium", "searchable", "image_url", "person_mongo_id", "gender", "diaspora_handle", "birthday", "last_name", "bio", "image_url_small", "first_name"]},
{:name => :posts, {:name => :posts,
:attrs => ["youtube_titles", "pending", "created_at", "public", "updated_at", "status_message_mongo_id", "caption", "remote_photo_path", "random_string", "image", "mongo_id", "type", "diaspora_handle", "person_mongo_id", "message"], :attrs => ["youtube_titles", "pending", "created_at", "public", "updated_at", "status_message_mongo_id", "caption", "remote_photo_path", "random_string", "image", "mongo_id", "type", "diaspora_handle", "person_mongo_id", "message"],
:mongo_attrs => ["youtube_titles", "pending", "created_at", "public", "updated_at", "status_message_id" , "caption", "remote_photo_path", "random_string", "image", "_id" , "_type", "diaspora_handle", "person_id" , "message"] :mongo_attrs => ["youtube_titles", "pending", "created_at", "public", "updated_at", "status_message_id" , "caption", "remote_photo_path", "random_string", "image", "_id" , "_type", "diaspora_handle", "person_id" , "message"]},
},
{:name => :requests, {:name => :requests,
:attrs => ["mongo_id", "recipient_mongo_id", "sender_mongo_id", "aspect_mongo_id"], :attrs => ["mongo_id", "recipient_mongo_id", "sender_mongo_id", "aspect_mongo_id"],
:mongo_attrs => ["_id" , "to_id" , "from_id" , "into_id"] :mongo_attrs => ["_id" , "to_id" , "from_id" , "into_id"]},
}, {:name => :services,
:attrs => ["type", "user_mongo_id", "provider", "uid", "access_token", "access_secret", "nickname"],
:mongo_attrs => ["_type", "user_id", "provider", "uid", "access_token", "access_secret", "nickname"]},
{:name => :users, {:name => :users,
:attrs => ["mongo_id", "username", "serialized_private_key", "encrypted_password", "invites", "invitation_token", "invitation_sent_at", "getting_started", "disable_mail", "language", "last_sign_in_ip", "last_sign_in_at", "reset_password_token", "password_salt"], :attrs => ["mongo_id", "username", "serialized_private_key", "encrypted_password", "invites", "invitation_token", "invitation_sent_at", "getting_started", "disable_mail", "language", "last_sign_in_ip", "last_sign_in_at", "reset_password_token", "password_salt"],
:mongo_attrs => ["_id", "username", "serialized_private_key", "encrypted_password", "invites", "invitation_token", "invitation_sent_at", "getting_started", "disable_mail", "language", "last_sign_in_ip", "last_sign_in_at", "reset_password_token", "password_salt"] :mongo_attrs => ["_id", "username", "serialized_private_key", "encrypted_password", "invites", "invitation_token", "invitation_sent_at", "getting_started", "disable_mail", "language", "last_sign_in_ip", "last_sign_in_at", "reset_password_token", "password_salt"]},
},
] ]
end end
def log(message) def log(message)

View file

@ -109,7 +109,11 @@ module DataConversion
model_hash[:mongo_attrs].map { |attr_name| hash[attr_name] } model_hash[:mongo_attrs].map { |attr_name| hash[attr_name] }
end end
end end
def services_json_to_csv model_hash
generic_json_to_csv(model_hash) do |hash|
model_hash[:mongo_attrs].map { |attr_name| hash[attr_name] }
end
end
def people_json_to_csv model_hash def people_json_to_csv model_hash
#Also writes the profiles csv #Also writes the profiles csv

View file

@ -92,6 +92,16 @@ module DataConversion
log "Finished. Imported #{Mongo::Contact.count} contacts." log "Finished. Imported #{Mongo::Contact.count} contacts."
end end
def import_raw_services
log "Loading services file..."
Mongo::Service.connection.execute <<-SQL
#{load_string("services")}
#{infile_opts}
(type,user_mongo_id,provider,uid,access_token,access_secret,nickname)
SQL
log "Finished. Imported #{Mongo::Service.count} services."
end
def import_raw_post_visibilities def import_raw_post_visibilities
log "Loading post visibilities file..." log "Loading post visibilities file..."
Mongo::PostVisibility.connection.execute <<-SQL Mongo::PostVisibility.connection.execute <<-SQL

View file

@ -0,0 +1,3 @@
type,user_mongo_id,provider,uid,access_token,access_secret,nickname
Services::Facebook,4d2657eacc8cb46033000011,,,yeah,,
Services::Twitter,4d2657e9cc8cb46033000009,,,yeah,foobar,
1 type user_mongo_id provider uid access_token access_secret nickname
2 Services::Facebook 4d2657eacc8cb46033000011 yeah
3 Services::Twitter 4d2657e9cc8cb46033000009 yeah foobar

View file

@ -233,7 +233,29 @@ describe DataConversion::ImportToMysql do
request.aspect_mongo_id.should == '' request.aspect_mongo_id.should == ''
end end
end end
describe "services" do
before do
copy_fixture_for("services")
end
it "imports data into the mongo_services table" do
Mongo::Service.count.should == 0
@migrator.import_raw_services
Mongo::Service.count.should == 2
end
it "imports all the columns" do
@migrator.import_raw_services
service = Mongo::Service.first
service.type.should == "Services::Facebook"
service.user_mongo_id.should == "4d2657eacc8cb46033000011"
service.provider.should be_nil
service.uid.should be_nil
service.access_token.should == "yeah"
service.access_secret.should be_nil
service.nickname.should be_nil
end
end
describe "users" do describe "users" do
before do before do
copy_fixture_for("users") copy_fixture_for("users")