From 86e52b0799f69a9a3c9c59eb84f7f925a88fd297 Mon Sep 17 00:00:00 2001
From: Ilya Zhitomirskiy <iz268@nyu.edu>
Date: Tue, 11 Oct 2011 12:03:35 -0700
Subject: [PATCH] remove statstample; made our own correlation function

---
 Gemfile           |  4 ----
 Gemfile.lock      | 41 -----------------------------------------
 lib/statistics.rb | 36 ++++++++++++++++++++++++++++++------
 3 files changed, 30 insertions(+), 51 deletions(-)

diff --git a/Gemfile b/Gemfile
index d08a2d3a1..a32c2ca19 100644
--- a/Gemfile
+++ b/Gemfile
@@ -77,10 +77,6 @@ gem 'SystemTimer', '1.2.1', :platforms => :ruby_18
 gem 'hoptoad_notifier'
 gem 'newrelic_rpm', :require => false
 
-# statistics
-
-gem 'statsample', :require => false
-
 #mail
 
 gem 'messagebus_ruby_api', '0.4.8'
diff --git a/Gemfile.lock b/Gemfile.lock
index 8a33b662c..68bab4162 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -139,7 +139,6 @@ GEM
       uuidtools
     childprocess (0.2.2)
       ffi (~> 1.0.6)
-    clbustos-rtf (0.4.2)
     closure-compiler (1.1.4)
     cloudfiles (1.4.10)
       mime-types (>= 1.16)
@@ -164,15 +163,12 @@ GEM
       devise (~> 1.3.1)
       rails (<= 3.2, >= 3.0.0)
     diff-lcs (1.1.3)
-    dirty-memoize (0.0.4)
-    distribution (0.6.0)
     em-synchrony (0.2.0)
       eventmachine (>= 0.12.9)
     erubis (2.6.6)
       abstract (>= 1.0.0)
     eventmachine (0.12.10)
     excon (0.2.4)
-    extendmatrix (0.3.1)
     extlib (0.9.15)
     factory_girl (2.1.2)
       activesupport
@@ -253,8 +249,6 @@ GEM
     mime-types (1.16)
     mini_magick (3.2)
       subexec (~> 0.0.4)
-    minimization (0.2.1)
-      text-table (~> 1.2)
     mixlib-authentication (1.1.4)
       mixlib-log
     mixlib-cli (1.2.2)
@@ -337,16 +331,6 @@ GEM
     parallel_tests (0.6.7)
       parallel
     polyglot (0.3.2)
-    prawn (0.8.4)
-      prawn-core (< 0.9, >= 0.8.4)
-      prawn-layout (< 0.9, >= 0.8.4)
-      prawn-security (< 0.9, >= 0.8.4)
-    prawn-core (0.8.4)
-    prawn-layout (0.8.4)
-    prawn-security (0.8.4)
-    prawn-svg (0.9.1.10)
-      prawn (>= 0.8.4)
-      prawn-core (>= 0.8.4)
     pyu-ruby-sasl (0.0.3.3)
     rack (1.2.4)
     rack-mobile-detect (0.3.0)
@@ -382,11 +366,6 @@ GEM
     redis (2.2.2)
     redis-namespace (0.8.0)
       redis (< 3.0.0)
-    reportbuilder (1.4.1)
-      clbustos-rtf (~> 0.4.0)
-      prawn (~> 0.8.4)
-      prawn-svg (~> 0.9.1)
-      text-table (~> 1.2)
     resque (1.10.0)
       json (~> 1.4.6)
       redis-namespace (~> 0.8.0)
@@ -399,7 +378,6 @@ GEM
       resque (~> 1.0)
     rest-client (1.6.1)
       mime-types (>= 1.16)
-    rserve-client (0.2.5)
     rspec (2.6.0)
       rspec-core (~> 2.6.0)
       rspec-expectations (~> 2.6.0)
@@ -428,7 +406,6 @@ GEM
       linecache19 (>= 0.5.11)
       ruby-debug-base19 (>= 0.11.19)
     ruby-hmac (0.4.0)
-    ruby-ole (1.2.11.2)
     ruby-openid (2.1.8)
     ruby-openid-apps-discovery (1.2.0)
       ruby-openid (>= 2.1.7)
@@ -436,7 +413,6 @@ GEM
     ruby_core_source (0.1.5)
       archive-tar-minitar (>= 0.5.2)
     rubyntlm (0.1.1)
-    rubyvis (0.4.1)
     rubyzip (0.9.4)
     sass (3.1.7)
     selenium-webdriver (2.7.0)
@@ -449,26 +425,10 @@ GEM
     sinatra (1.2.7)
       rack (~> 1.1)
       tilt (>= 1.2.2, < 2.0)
-    spreadsheet (0.6.5.9)
-      ruby-ole (>= 1.0)
     sqlite3 (1.3.4)
-    statsample (1.1.0)
-      dirty-memoize (~> 0.0)
-      distribution (~> 0.3)
-      extendmatrix (~> 0.3.1)
-      fastercsv (> 0)
-      minimization (~> 0.2.0)
-      reportbuilder (~> 1.4)
-      rserve-client (~> 0.2.5)
-      rubyvis (~> 0.4.0)
-      spreadsheet (~> 0.6.5)
-      statsample-bivariate-extension (> 0)
-    statsample-bivariate-extension (1.1.0)
-      distribution (~> 0.6)
     subexec (0.0.4)
     systemu (2.4.0)
     term-ansicolor (1.0.6)
-    text-table (1.2.2)
     thin (1.2.11)
       daemons (>= 1.0.9)
       eventmachine (>= 0.12.6)
@@ -577,7 +537,6 @@ DEPENDENCIES
   settingslogic (= 2.0.6)
   sod!
   sqlite3
-  statsample
   thin (= 1.2.11)
   twitter (= 1.5.0)
   typhoeus
diff --git a/lib/statistics.rb b/lib/statistics.rb
index 2cef34652..8219a5c8a 100644
--- a/lib/statistics.rb
+++ b/lib/statistics.rb
@@ -1,5 +1,3 @@
-require 'statsample'
-
 class Statistics
 
   attr_reader :start_time,
@@ -109,10 +107,36 @@ SQL
   end
 
   def correlation(x_array, y_array)
-    x = x_array.to_scale
-    y = y_array.to_scale
-    pearson = Statsample::Bivariate::Pearson.new(x,y)
-    pearson.r
+    sum = 0.0
+    x_array.each_index do |i|
+      sum = sum + x_array[i]*y_array[i]
+    end
+    x_y_mean = sum/x_array.length.to_f
+    x_mean = mean(x_array)
+    y_mean = mean(y_array)
+
+    st_dev_x = standard_deviation(x_array)
+    st_dev_y = standard_deviation(y_array)
+
+    (x_y_mean - (x_mean*y_mean))/(st_dev_x * st_dev_y)
+  end
+
+  def mean(array)
+    sum = array.inject(0.0) do |sum, val|
+      sum += val
+    end
+    sum / array.length
+  end
+
+  def standard_deviation(array)
+    variance = lambda do
+      m = mean(array)
+      sum = 0.0
+      array.each{ |v| sum += (v-m)**2 }
+      sum/array.length.to_f
+    end.call
+
+    Math.sqrt(variance)
   end
 
   ### % of cohort came back last week