From 0e7521df9fc3386340331b27076d8b6b19d214e8 Mon Sep 17 00:00:00 2001
From: Maxwell Salzberg <maxwell@joindiaspora.com>
Date: Sat, 3 Sep 2011 16:07:06 -0700
Subject: [PATCH 1/6] adding csv generator

---
 lib/csv_generator.rb           | 69 ++++++++++++++++++++++++++++++++++
 spec/lib/csv_generator_spec.rb | 19 ++++++++++
 2 files changed, 88 insertions(+)
 create mode 100644 lib/csv_generator.rb
 create mode 100644 spec/lib/csv_generator_spec.rb

diff --git a/lib/csv_generator.rb b/lib/csv_generator.rb
new file mode 100644
index 000000000..de12bccf2
--- /dev/null
+++ b/lib/csv_generator.rb
@@ -0,0 +1,69 @@
+module CsvGenerator
+
+  PATH = '/Users/maxwell/Sites/dump'
+
+  def self.all_active_users
+    file = self.filename("all_active_users")
+    sql = <<SQL
+      SELECT email AS '%EMAIL%' 
+        #{self.output_syntax(file)}
+        FROM users where username IS NOT NULL
+SQL
+    ActiveRecord::Base.execute(sql)
+  end
+
+  def self.all_inactive_invited_users
+    file = self.filename("all_inactive_invited_users.csv")
+    sql = <<SQL
+      SELECT invitations.identifier AS '%EMAIL%', users.invitation_token AS '%TOKEN%'
+        #{self.output_syntax(file)}
+        FROM invitations
+        JOIN users ON
+          users.id=invitations.recipient_id
+          WHERE users.username IS NULL
+            AND invitations.service='email'
+SQL
+    ActiveRecord::Base.execute(sql)
+  end
+
+  def self.waitlist
+    filename = File.join(Rails.root, 'config', 'mailing_list.csv')
+
+    people = self.load_waiting_list_csv
+    offset = self.offset
+    left = people[0...offset]
+    right = people[offset...people.size]
+
+    #reading from csv (get number of row we're on) - left
+    #reading from csv (get number of row we're on) - right
+  end
+
+  def self.load_waiting_list_csv
+    csv= File.join(Rails.root, 'config', 'mailing_list.csv')
+    if RUBY_VERSION.include? "1.8"
+      require 'fastercsv'
+       people = FasterCSV.read(csv)
+     else
+       require 'csv'
+       people = CSV.read(csv)
+     end
+    people
+  end
+
+  def self.offset
+    offset_filename = File.join(Rails.root, 'config', 'email_offset')
+    File.read(offset_filename).to_i
+  end
+
+  def self.filename(name)
+    "#{PATH}#{Time.now.strftime("%Y-%m-%d")}-#{name}"
+  end
+
+  def self.output_syntax filename
+    <<SQL
+    INTO OUTFILE '#{filename}'
+FIELDS TERMINATED BY '\t'
+LINES TERMINATED BY '\n'
+SQL
+  end
+end
diff --git a/spec/lib/csv_generator_spec.rb b/spec/lib/csv_generator_spec.rb
new file mode 100644
index 000000000..f8068da5b
--- /dev/null
+++ b/spec/lib/csv_generator_spec.rb
@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+describe CsvGenerator do
+  describe '.all_active_users' do
+
+  end
+
+  describe '.all_inactive_invited_users' do
+
+  end
+
+  describe '.waitlist_sent' do
+
+  end
+
+  describe '.waitlist_pending' do
+
+  end
+end

From 748527157383e02e81ab97edcdd883bcd7518b52 Mon Sep 17 00:00:00 2001
From: Ilyaaaaaaaaaaaaa Zhitomirskiy <ilya@joindiaspora.com>
Date: Tue, 6 Sep 2011 15:47:18 -0700
Subject: [PATCH 2/6] wip csv gen

---
 lib/csv_generator.rb | 90 +++++++++++++++++++++++++++++++++++++-------
 1 file changed, 76 insertions(+), 14 deletions(-)

diff --git a/lib/csv_generator.rb b/lib/csv_generator.rb
index de12bccf2..fb281260b 100644
--- a/lib/csv_generator.rb
+++ b/lib/csv_generator.rb
@@ -1,6 +1,9 @@
 module CsvGenerator
 
-  PATH = '/Users/maxwell/Sites/dump'
+  PATH = '/usr/local/app/diaspora'
+  BACKER_CSV_LOCATION = File.join('/usr/local/app/diaspora/', 'backer_list.csv')
+  WAITLIST_LOCATION = File.join(Rails.root, 'config', 'mailing_list.csv')
+  OFFSET_LOCATION = File.join(Rails.root, 'config', 'email_offset')
 
   def self.all_active_users
     file = self.filename("all_active_users")
@@ -26,20 +29,61 @@ SQL
     ActiveRecord::Base.execute(sql)
   end
 
-  def self.waitlist
-    filename = File.join(Rails.root, 'config', 'mailing_list.csv')
-
-    people = self.load_waiting_list_csv
-    offset = self.offset
-    left = people[0...offset]
-    right = people[offset...people.size]
-
-    #reading from csv (get number of row we're on) - left
-    #reading from csv (get number of row we're on) - right
+  def self.backers_recent_login
+    file = self.filename("v1_backers_recent_login.csv")
+    sql = <<SQL
+          SELECT email AS '%EMAIL%',
+                 coalesce( full_name, 'friend of Diaspora*') AS '%NAME%',
+                 invitation_token AS '%TOKEN%'
+              #{self.output_syntax(file)}
+           FROM `users` 
+          WHERE #{self.backer_email_condition}
+            AND (last_sign_in_at >= #{(Time.now - 1.month).to_i})
+SQL
   end
 
-  def self.load_waiting_list_csv
-    csv= File.join(Rails.root, 'config', 'mailing_list.csv')
+  def self.backers_older_login
+    file = self.filename("v1_backers_recent_login.csv")
+    sql = <<SQL
+          SELECT email AS '%EMAIL%',
+                 coalesce( full_name, 'friend of Diaspora*') AS '%NAME%',
+                 invitation_token AS '%TOKEN%'
+              #{self.output_syntax(file)}
+           FROM `users` 
+          WHERE #{self.backer_email_condition}
+            AND (last_sign_in_at < #{(Time.now - 1.month).to_i})
+SQL
+  end
+
+
+  # ---------------- QUERY METHODS & NOTES -------------------------
+  def self.backer_email_condition
+    "`users`.`email` IN (#{query_string_from_array(self.backer_emails)})" 
+  end
+
+  def self.recent_login_query
+
+  end
+  
+  def self.query_string_from_array(array)
+    array.join(", ")
+  end
+  
+  # BACKER RECENT LOGIN
+  # User.where("last_sign_in_at > ?", (Time.now - 1.month).to_i).where(:email => ["maxwell@joindiaspora.com"]).count
+  #
+  # "SELECT `users`.* FROM `users` WHERE `users`.`email` IN ('maxwell@joindiaspora.com') AND (last_sign_in_at > 1312663724)"
+
+  # NON BACKER RECENT LOGIN
+  # User.where("last_sign_in_at > ?", (Time.now - 1.month).to_i).where("email NOT IN (?)", 'maxwell@joindiaspora.com').to_sql
+  # "SELECT `users`.* FROM `users` WHERE (last_sign_in_at > 1312665370) AND (email NOT IN ('maxwell@joindiaspora.com'))" 
+ 
+
+
+
+  # ---------------- HELPER METHODS -------------------------
+  def self.load_waiting_list_csv(filename)
+    csv = filename
     if RUBY_VERSION.include? "1.8"
       require 'fastercsv'
        people = FasterCSV.read(csv)
@@ -51,7 +95,7 @@ SQL
   end
 
   def self.offset
-    offset_filename = File.join(Rails.root, 'config', 'email_offset')
+    offset_filename = OFFSET_LOCATION
     File.read(offset_filename).to_i
   end
 
@@ -66,4 +110,22 @@ FIELDS TERMINATED BY '\t'
 LINES TERMINATED BY '\n'
 SQL
   end
+
+  def self.waitlist
+    people = self.load_waiting_list_csv(WAITLIST_LOCATION)
+    offset = self.offset
+    left = people[0...offset]
+    right = people[offset...people.size]
+
+    #reading from csv (get number of row we're on) - left
+    #reading from csv (get number of row we're on) - right
+  end
+
+  def self.backers
+    self.load_waiting_list_csv(BACKER_CSV_LOCATION)
+  end
+
+  def self.backer_emails
+    self.backers.map{|b| b[0]}
+  end
 end

From 91d60f5e01817f1209ece6ce7164ba1481fef2ca Mon Sep 17 00:00:00 2001
From: Ilyaaaaaaaaaaaaa Zhitomirskiy <ilya@joindiaspora.com>
Date: Tue, 6 Sep 2011 22:58:52 -0700
Subject: [PATCH 3/6] wip

---
 lib/csv_generator.rb | 51 +++++++++++++++++++++++++++-----------------
 1 file changed, 31 insertions(+), 20 deletions(-)

diff --git a/lib/csv_generator.rb b/lib/csv_generator.rb
index fb281260b..68ac7f52d 100644
--- a/lib/csv_generator.rb
+++ b/lib/csv_generator.rb
@@ -1,7 +1,7 @@
 module CsvGenerator
 
-  PATH = '/usr/local/app/diaspora'
-  BACKER_CSV_LOCATION = File.join('/usr/local/app/diaspora/', 'backer_list.csv')
+  PATH = '/home/ilya/workspace/diaspora/'
+  BACKER_CSV_LOCATION = File.join('/home/ilya/workspace/diaspora/', 'backer_list.csv')
   WAITLIST_LOCATION = File.join(Rails.root, 'config', 'mailing_list.csv')
   OFFSET_LOCATION = File.join(Rails.root, 'config', 'email_offset')
 
@@ -31,34 +31,45 @@ SQL
 
   def self.backers_recent_login
     file = self.filename("v1_backers_recent_login.csv")
+    # coalesce( `profiles`.full_name, )
+    # JOIN ( `profiles`.)
     sql = <<SQL
-          SELECT email AS '%EMAIL%',
-                 coalesce( full_name, 'friend of Diaspora*') AS '%NAME%',
-                 invitation_token AS '%TOKEN%'
+          SELECT `users`.email AS '%EMAIL%',
+                 'friend of Diaspora*' AS '%NAME%',
+                 `users`.invitation_token AS '%TOKEN%'
               #{self.output_syntax(file)}
-           FROM `users` 
+           FROM `users`
           WHERE #{self.backer_email_condition}
-            AND (last_sign_in_at >= #{(Time.now - 1.month).to_i})
+            AND (last_sign_in_at > 1312675027);
 SQL
+
+    puts "Here is the sql:"
+    puts sql
+    puts
+
+    User.connection.execute(sql)
   end
 
-  def self.backers_older_login
-    file = self.filename("v1_backers_recent_login.csv")
-    sql = <<SQL
-          SELECT email AS '%EMAIL%',
-                 coalesce( full_name, 'friend of Diaspora*') AS '%NAME%',
-                 invitation_token AS '%TOKEN%'
-              #{self.output_syntax(file)}
-           FROM `users` 
-          WHERE #{self.backer_email_condition}
-            AND (last_sign_in_at < #{(Time.now - 1.month).to_i})
-SQL
-  end
+#  def self.backers_older_login
+#    file = self.filename("v1_backers_recent_login.csv")
+#    sql = <<SQL
+#          SELECT email AS '%EMAIL%',
+#                 coalesce( full_name, 'friend of Diaspora*') AS '%NAME%',
+#                 invitation_token AS '%TOKEN%'
+#              #{self.output_syntax(file)}
+#           FROM `users` 
+#          WHERE #{self.backer_email_condition}
+#            AND (last_sign_in_at < #{(Time.now - 1.month).to_i})
+#SQL
+#    ActiveRecord::Base.execute(sql)
+#  end
 
 
   # ---------------- QUERY METHODS & NOTES -------------------------
   def self.backer_email_condition
-    "`users`.`email` IN (#{query_string_from_array(self.backer_emails)})" 
+    b_emails = self.backer_emails
+    b_emails.map!{|a| "'#{a}'"}
+    "`users`.`email` IN (#{query_string_from_array(b_emails[1..b_emails.length])})" 
   end
 
   def self.recent_login_query

From c03fec1244eab3e81dab1fa748f44c23b75c94d0 Mon Sep 17 00:00:00 2001
From: Ilyaaaaaaaaaaaaa Zhitomirskiy <ilya@joindiaspora.com>
Date: Wed, 7 Sep 2011 01:47:35 -0700
Subject: [PATCH 4/6] the non-waitlist csv writing to tmp

---
 lib/csv_generator.rb | 112 ++++++++++++++++++++++++++++++-------------
 1 file changed, 79 insertions(+), 33 deletions(-)

diff --git a/lib/csv_generator.rb b/lib/csv_generator.rb
index 68ac7f52d..c4e5d320f 100644
--- a/lib/csv_generator.rb
+++ b/lib/csv_generator.rb
@@ -1,6 +1,6 @@
 module CsvGenerator
 
-  PATH = '/home/ilya/workspace/diaspora/'
+  PATH = '/tmp/'
   BACKER_CSV_LOCATION = File.join('/home/ilya/workspace/diaspora/', 'backer_list.csv')
   WAITLIST_LOCATION = File.join(Rails.root, 'config', 'mailing_list.csv')
   OFFSET_LOCATION = File.join(Rails.root, 'config', 'email_offset')
@@ -12,7 +12,7 @@ module CsvGenerator
         #{self.output_syntax(file)}
         FROM users where username IS NOT NULL
 SQL
-    ActiveRecord::Base.execute(sql)
+    ActiveRecord::Base.connection.execute(sql)
   end
 
   def self.all_inactive_invited_users
@@ -26,54 +26,100 @@ SQL
           WHERE users.username IS NULL
             AND invitations.service='email'
 SQL
-    ActiveRecord::Base.execute(sql)
+    ActiveRecord::Base.connection.execute(sql)
+  end
+
+  def self.generate_csvs
+    self.backers_recent_login
+    self.backers_old_login
+    self.backers_never_login
+    self.non_backers_recent_login
+    self.non_backers_old_login
+    self.non_backers_never_login
   end
 
   def self.backers_recent_login
     file = self.filename("v1_backers_recent_login.csv")
-    # coalesce( `profiles`.full_name, )
-    # JOIN ( `profiles`.)
-    sql = <<SQL
-          SELECT `users`.email AS '%EMAIL%',
-                 'friend of Diaspora*' AS '%NAME%',
-                 `users`.invitation_token AS '%TOKEN%'
-              #{self.output_syntax(file)}
-           FROM `users`
-          WHERE #{self.backer_email_condition}
-            AND (last_sign_in_at > 1312675027);
-SQL
+    sql = self.select_fragment(file, "#{self.backer_email_condition} AND #{self.recent_login_query}")
 
-    puts "Here is the sql:"
-    puts sql
-    puts
-
-    User.connection.execute(sql)
+    ActiveRecord::Base.connection.execute(sql)
   end
 
-#  def self.backers_older_login
-#    file = self.filename("v1_backers_recent_login.csv")
-#    sql = <<SQL
-#          SELECT email AS '%EMAIL%',
-#                 coalesce( full_name, 'friend of Diaspora*') AS '%NAME%',
-#                 invitation_token AS '%TOKEN%'
-#              #{self.output_syntax(file)}
-#           FROM `users` 
-#          WHERE #{self.backer_email_condition}
-#            AND (last_sign_in_at < #{(Time.now - 1.month).to_i})
-#SQL
-#    ActiveRecord::Base.execute(sql)
-#  end
+  def self.backers_old_login
+    file = self.filename("v2_backers_old_login.csv")
+    sql = self.select_fragment(file, "#{self.backer_email_condition} AND #{self.old_login_query}")
 
+    ActiveRecord::Base.connection.execute(sql)
+  end
+
+  def self.backers_never_login
+    file = self.filename("v3_backers_never_login.csv")
+    sql = self.select_fragment(file, "#{self.backer_email_condition} AND #{self.never_login_query}")
+
+    ActiveRecord::Base.connection.execute(sql)
+  end
+
+  def self.non_backers_recent_login
+    file = self.filename("v4_non_backers_recent_login.csv")
+    sql = self.select_fragment(file, "#{self.non_backer_email_condition} AND #{self.recent_login_query}")
+
+    ActiveRecord::Base.connection.execute(sql)
+  end
+
+  def self.non_backers_old_login
+    file = self.filename("v5_non_backers_old_login.csv")
+    sql = self.select_fragment(file, "#{self.non_backer_email_condition} AND #{self.old_login_query}")
+
+    ActiveRecord::Base.connection.execute(sql)
+  end
+
+  def self.non_backers_never_login
+    file = self.filename("v6_non_backers_never_login.csv")
+    sql = self.select_fragment(file, "#{self.non_backer_email_condition} AND #{self.never_login_query}") #
+    ActiveRecord::Base.connection.execute(sql)
+  end
 
   # ---------------- QUERY METHODS & NOTES -------------------------
+  def self.select_fragment(file, where_clause)
+    #    #
+    #
+    #
+    sql = <<SQL
+          SELECT '%EMAIL%','%NAME%','%INVITATION_LINK%'
+          UNION
+            SELECT `users`.email AS '%EMAIL%',
+                   IF( `profiles`.full_name IS NOT NULL AND `profiles`.full_name != "",
+                                               `profiles`.full_name, 'friend of Diaspora*') AS '%NAME%',
+                IF(`users`.invitation_token, CONCAT( 'http://joindiaspora.com/users/invitation/accept?invitation_token=', `users`.invitation_token) ,NULL) AS '%INVITATION_LINK%'
+                #{self.output_syntax(file)}
+             FROM `users`
+             JOIN `people` ON `users`.id = `people`.owner_id JOIN `profiles` ON `people`.id = `profiles`.person_id
+            WHERE #{where_clause};
+SQL
+  end
+
   def self.backer_email_condition
     b_emails = self.backer_emails
     b_emails.map!{|a| "'#{a}'"}
     "`users`.`email` IN (#{query_string_from_array(b_emails[1..b_emails.length])})" 
   end
 
-  def self.recent_login_query
+  def self.non_backer_email_condition
+    b_emails = self.backer_emails
+    b_emails.map!{|a| "'#{a}'"}
+    "`users`.`email` NOT IN (#{query_string_from_array(b_emails[1..b_emails.length])})" 
+  end
 
+  def self.recent_login_query
+    "(last_sign_in_at > #{(Time.now - 1.month).to_i})"
+  end
+
+  def self.old_login_query
+    "(last_sign_in_at < #{(Time.now - 1.month).to_i})"
+  end
+
+  def self.never_login_query
+    "(last_sign_in_at IS NULL)"
   end
   
   def self.query_string_from_array(array)

From ef9eff300b6e6c22aaa4bc394f94bacc96580922 Mon Sep 17 00:00:00 2001
From: Ilyaaaaaaaaaaaaa Zhitomirskiy <ilya@joindiaspora.com>
Date: Wed, 7 Sep 2011 02:44:09 -0700
Subject: [PATCH 5/6] not doing joins where they are not needed

---
 lib/csv_generator.rb | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/lib/csv_generator.rb b/lib/csv_generator.rb
index c4e5d320f..2e671fc49 100644
--- a/lib/csv_generator.rb
+++ b/lib/csv_generator.rb
@@ -54,7 +54,16 @@ SQL
 
   def self.backers_never_login
     file = self.filename("v3_backers_never_login.csv")
-    sql = self.select_fragment(file, "#{self.backer_email_condition} AND #{self.never_login_query}")
+    sql = <<SQL
+          SELECT '%EMAIL%','%NAME%','%INVITATION_LINK%'
+          UNION
+            SELECT `users`.email AS '%EMAIL%',
+                    'friend of Diaspora*' AS '%NAME%',
+                IF(`users`.invitation_token, CONCAT( 'http://joindiaspora.com/users/invitation/accept?invitation_token=', `users`.invitation_token) ,NULL) AS '%INVITATION_LINK%'
+                #{self.output_syntax(file)}
+             FROM `users`
+            WHERE #{self.backer_email_condition} AND #{self.never_login_query};
+SQL
 
     ActiveRecord::Base.connection.execute(sql)
   end
@@ -75,7 +84,16 @@ SQL
 
   def self.non_backers_never_login
     file = self.filename("v6_non_backers_never_login.csv")
-    sql = self.select_fragment(file, "#{self.non_backer_email_condition} AND #{self.never_login_query}") #
+    sql = <<SQL
+          SELECT '%EMAIL%','%NAME%','%INVITATION_LINK%'
+          UNION
+            SELECT `users`.email AS '%EMAIL%',
+                    'friend of Diaspora*' AS '%NAME%',
+                IF(`users`.invitation_token, CONCAT( 'http://joindiaspora.com/users/invitation/accept?invitation_token=', `users`.invitation_token) ,NULL) AS '%INVITATION_LINK%'
+                #{self.output_syntax(file)}
+             FROM `users`
+            WHERE #{self.non_backer_email_condition} AND #{self.never_login_query};
+SQL
     ActiveRecord::Base.connection.execute(sql)
   end
 

From b76133c56a08e7ee0d8154f26b525d1dfc781321 Mon Sep 17 00:00:00 2001
From: Ilyaaaaaaaaaaaaa Zhitomirskiy <ilya@joindiaspora.com>
Date: Wed, 7 Sep 2011 02:49:35 -0700
Subject: [PATCH 6/6] separate tmp directory

---
 lib/csv_generator.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/csv_generator.rb b/lib/csv_generator.rb
index 2e671fc49..a155a60c5 100644
--- a/lib/csv_generator.rb
+++ b/lib/csv_generator.rb
@@ -1,6 +1,6 @@
 module CsvGenerator
 
-  PATH = '/tmp/'
+  PATH = '/tmp/csvs'
   BACKER_CSV_LOCATION = File.join('/home/ilya/workspace/diaspora/', 'backer_list.csv')
   WAITLIST_LOCATION = File.join(Rails.root, 'config', 'mailing_list.csv')
   OFFSET_LOCATION = File.join(Rails.root, 'config', 'email_offset')
@@ -30,6 +30,7 @@ SQL
   end
 
   def self.generate_csvs
+    `mkdir /tmp/csvs`
     self.backers_recent_login
     self.backers_old_login
     self.backers_never_login