From 3df36721d6582fedd1293d06746bf9a09ab3b317 Mon Sep 17 00:00:00 2001 From: Raphael Date: Thu, 6 Jan 2011 10:31:45 -0800 Subject: [PATCH] Move attribute lists out to base, so they are together and we can use them in the import --- lib/data_conversion/base.rb | 41 +++++++++++++++++- lib/data_conversion/export_from_mongo.rb | 53 ++++-------------------- 2 files changed, 47 insertions(+), 47 deletions(-) diff --git a/lib/data_conversion/base.rb b/lib/data_conversion/base.rb index 64abe9a9d..6823e56cf 100644 --- a/lib/data_conversion/base.rb +++ b/lib/data_conversion/base.rb @@ -11,7 +11,44 @@ module DataConversion @directory = "tmp/export-for-mysql" @full_path = "#{Rails.root}/#{directory}" end - + def models + @models ||= [ + {:name => :aspects, + :main_attrs => ["mongo_id", "name", "user_mongo_id", "created_at", "updated_at"], + :mongo_attrs => ["_id" , "name", "user_id" , "created_at", "updated_at"], + :join_table_name => :post_visibilities, + :join_table_attrs => ["aspect_mongo_id", "post_mongo_id"]}, + {:name => :comments, + :attrs => ["mongo_id", "post_mongo_id", "person_mongo_id", "diaspora_handle", "text", "youtube_titles"], + :mongo_attrs => ["_id", "post_id", "person_id", "diaspora_handle", "text", "youtube_titles"]}, + {:name => :contacts, + :main_attrs => ["mongo_id", "user_mongo_id", "person_mongo_id", "pending", "created_at", "updated_at"], + :main_mongo_attrs => ["_id" , "user_id" , "person_id" , "pending", "created_at", "updated_at"], + :join_table_name => :aspect_memberships, + :join_table_attrs => ["contact_mongo_id", "aspect_mongo_id"]}, + {:name => :invitations, + :attrs => ["mongo_id", "recipient_mongo_id", "sender_mongo_id", "aspect_mongo_id", "message"], + :mongo_attrs => ["_id" , "to_id" , "from_id" , "into_id" , "message"]}, + {:name => :notifications, + :attrs => ["mongo_id", "target_id", "target_type", "unread"], + :mongo_attrs => ["_id" , "target_id", "kind" , "unread"]}, + {:name => :people, + :attrs => ["created_at", "updated_at", "serialized_public_key", "url", "mongo_id", "owner_mongo_id", "diaspora_handle"], + :profile_attrs => ["image_url_medium", "searchable", "image_url", "person_mongo_id", "gender", "diaspora_handle", "birthday", "last_name", "bio", "image_url_small", "first_name"]}, + {:name => :posts, + :attrs => ["youtube_titles", "pending", "created_at", "public", "updated_at", "status_message_mongo_id", "caption", "remote_photo_path", "random_string", "image", "mongo_id", "type", "diaspora_handle", "person_mongo_id", "message"], + :mongo_attrs => ["youtube_titles", "pending", "created_at", "public", "updated_at", "status_message_id" , "caption", "remote_photo_path", "random_string", "image", "_id" , "_type", "diaspora_handle", "person_id" , "message"] + }, + {:name => :requests, + :attrs => ["mongo_id", "recipient_mongo_id", "sender_mongo_id", "aspect_mongo_id"], + :mongo_attrs => ["_id" , "to_id" , "from_id" , "into_id"] + }, + {:name => :users, + :attrs => ["mongo_id", "username", "serialized_private_key", "encrypted_password", "invites", "invitation_token", "invitation_sent_at", "getting_started", "disable_mail", "language", "last_sign_in_ip", "last_sign_in_at", "reset_password_token", "password_salt"], + :mongo_attrs => ["_id", "username", "serialized_private_key", "encrypted_password", "invites", "invitation_token", "invitation_sent_at", "getting_started", "disable_mail", "language", "last_sign_in_ip", "last_sign_in_at", "reset_password_token", "password_salt"] + }, + ] + end def log(message) if ['development', 'production'].include?(Rails.env) puts "#{sprintf("%.2f", Time.now - start_time)}s #{message}" @@ -19,4 +56,4 @@ module DataConversion Rails.logger.debug(message) if Rails.logger end end -end \ No newline at end of file +end diff --git a/lib/data_conversion/export_from_mongo.rb b/lib/data_conversion/export_from_mongo.rb index 762db285c..68756a903 100644 --- a/lib/data_conversion/export_from_mongo.rb +++ b/lib/data_conversion/export_from_mongo.rb @@ -32,21 +32,7 @@ module DataConversion "diaspora-#{Rails.env}" end - def models - @models ||= [ - {:name => :aspects}, - {:name => :comments, - :attrs => ["mongo_id", "post_mongo_id", "person_mongo_id", "diaspora_handle", "text", "youtube_titles"], - :mongo_attrs => ["_id", "post_id", "person_id", "diaspora_handle", "text", "youtube_titles"]}, - {:name => :contacts}, - {:name => :invitations}, - {:name => :notifications}, - {:name => :people}, - {:name => :posts}, - {:name => :requests}, - {:name => :users}, - ] - end + def id_sed @id_sed = sed_replace('{\ \"$oid\"\ :\ \(\"[^"]*\"\)\ }') @@ -104,14 +90,8 @@ module DataConversion end def contacts_json_to_csv model_hash - model_hash[:main_attrs] = ["mongo_id", "user_mongo_id", "person_mongo_id", "pending", "created_at", "updated_at"] - #Post Visibilities - model_hash[:join_table_name] = :aspect_memberships - model_hash[:join_table_attrs] = ["contact_mongo_id", "aspect_mongo_id"] - generic_json_to_two_csvs(model_hash) do |hash| - main_mongo_attrs = ["_id", "user_id", "person_id", "pending", "created_at", "updated_at"] - main_row = main_mongo_attrs.map { |attr_name| hash[attr_name] } + main_row = model_hash[:main_mongo_attrs].map { |attr_name| hash[attr_name] } aspect_membership_rows = hash["aspect_ids"].map { |id| [hash["_id"], id] } [main_row, aspect_membership_rows] end @@ -119,24 +99,18 @@ module DataConversion end def invitations_json_to_csv model_hash - model_hash[:attrs] = ["mongo_id", "recipient_mongo_id", "sender_mongo_id", "aspect_mongo_id", "message"] generic_json_to_csv(model_hash) do |hash| - mongo_attrs = ["_id", "to_id", "from_id", "into_id", "message"] - mongo_attrs.map { |attr_name| hash[attr_name] } + model_hash[:mongo_attrs].map { |attr_name| hash[attr_name] } end end def notifications_json_to_csv model_hash - model_hash[:attrs] = ["mongo_id", "target_id", "target_type", "unread"] generic_json_to_csv(model_hash) do |hash| - mongo_attrs = ["_id", "target_id", "kind", "unread"] - mongo_attrs.map { |attr_name| hash[attr_name] } + model_hash[:mongo_attrs].map { |attr_name| hash[attr_name] } end end def people_json_to_csv model_hash - model_hash[:attrs] = ["created_at", "updated_at", "serialized_public_key", "url", "mongo_id", "owner_mongo_id", "diaspora_handle"] - model_hash[:profile_attrs] = ["image_url_medium", "searchable", "image_url", "person_mongo_id", "gender", "diaspora_handle", "birthday", "last_name", "bio", "image_url_small", "first_name"] #Also writes the profiles csv log "Converting #{model_hash[:name]} json to csv" @@ -170,40 +144,29 @@ module DataConversion end def posts_json_to_csv model_hash - model_hash[:attrs] =["youtube_titles", "pending", "created_at", "public", "updated_at", "status_message_mongo_id", "caption", "remote_photo_path", "random_string", "image", "mongo_id", "type", "diaspora_handle", "person_mongo_id", "message"] generic_json_to_csv(model_hash) do |hash| - mongo_attrs = ["youtube_titles", "pending", "created_at", "public", "updated_at", "status_message_id", "caption", "remote_photo_path", "random_string", "image", "_id", "_type", "diaspora_handle", "person_id", "message"] - mongo_attrs.map { |attr_name| hash[attr_name] } + model_hash[:mongo_attrs].map { |attr_name| hash[attr_name] } end #has to handle the polymorphic stuff end def requests_json_to_csv model_hash - model_hash[:attrs] = ["mongo_id", "recipient_mongo_id", "sender_mongo_id", "aspect_mongo_id"] generic_json_to_csv(model_hash) do |hash| - mongo_attrs = ["_id", "to_id", "from_id", "into_id"] - mongo_attrs.map { |attr_name| hash[attr_name] } + model_hash[:mongo_attrs].map { |attr_name| hash[attr_name] } end end def users_json_to_csv model_hash - model_hash[:attrs] = ["mongo_id", "username", "serialized_private_key", "encrypted_password", "invites", "invitation_token", "invitation_sent_at", "getting_started", "disable_mail", "language", "last_sign_in_ip", "last_sign_in_at", "reset_password_token", "password_salt"] generic_json_to_csv(model_hash) do |hash| - mongo_attrs = ["_id", "username", "serialized_private_key", "encrypted_password", "invites", "invitation_token", "invitation_sent_at", "getting_started", "disable_mail", "language", "last_sign_in_ip", "last_sign_in_at", "reset_password_token", "password_salt"] - mongo_attrs.map { |attr_name| hash[attr_name] } + model_hash[:mongo_attrs].map { |attr_name| hash[attr_name] } end end def aspects_json_to_csv model_hash log "Converting aspects json to aspects and post_visibilities csvs" - model_hash[:main_attrs] = ["mongo_id", "name", "user_mongo_id", "created_at", "updated_at"] - #Post Visibilities - model_hash[:join_table_name] = :post_visibilities - model_hash[:join_table_attrs] = ["aspect_mongo_id", "post_mongo_id"] generic_json_to_two_csvs(model_hash) do |hash| - mongo_attrs = ["_id", "name", "user_id", "created_at", "updated_at"] - main_row = mongo_attrs.map { |attr_name| hash[attr_name] } + main_row = model_hash[:mongo_attrs].map { |attr_name| hash[attr_name] } post_visibility_rows = hash["post_ids"].map { |id| [hash["_id"], id] } [main_row, post_visibility_rows] end