From c8f482064c734dbbe84dbb021cc65a45aa11b8de Mon Sep 17 00:00:00 2001 From: Steffen van Bergerem Date: Wed, 22 Jul 2015 00:58:02 +0200 Subject: [PATCH 1/2] Remove rake_helpers --- lib/rake_helpers.rb | 40 ------------------------------------ spec/lib/rake_helper_spec.rb | 26 ----------------------- 2 files changed, 66 deletions(-) delete mode 100644 lib/rake_helpers.rb delete mode 100644 spec/lib/rake_helper_spec.rb diff --git a/lib/rake_helpers.rb b/lib/rake_helpers.rb deleted file mode 100644 index 20284cb40..000000000 --- a/lib/rake_helpers.rb +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2010-2011, Diaspora Inc. This file is -# licensed under the Affero General Public License version 3 or later. See -# the COPYRIGHT file. -module RakeHelpers - - def process_emails(csv, num_to_process, offset, test=true) - require 'csv' - backers = CSV.read(csv) - puts "DRY RUN" if test - churn_through = 0 - num_to_process.times do |n| - if backers[n+offset] == nil - break - end - churn_through = n - backer_name = backers[n+offset][1].to_s.strip - backer_email = backers[n+offset][0].to_s.strip.downcase - - possible_user = User.find_by_email(backer_email) - possible_invite = Invitation.find_by_identifier(backer_email) - possible_user ||= possible_invite.recipient if possible_invite.present? - - admin_account = User.find_by_username(AppConfig.admins.account.get) - raise "no admin account in diaspora.yml" unless admin_account.present? - admin_account.invitation_code.count += num_to_process - admin_account.invitation_code.save - - unless possible_user - puts "#{n}: sending email to: #{backer_name} #{backer_email}" unless Rails.env == 'test' - unless test - i = EmailInviter.new(backer_email) - i.send! - end - else - puts "user with the email exists: #{backer_email} , #{backer_name} " unless Rails.env == 'test' - end - end - churn_through - end -end diff --git a/spec/lib/rake_helper_spec.rb b/spec/lib/rake_helper_spec.rb deleted file mode 100644 index c0db62701..000000000 --- a/spec/lib/rake_helper_spec.rb +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) 2010-2011, Diaspora Inc. This file is -# licensed under the Affero General Public License version 3 or later. See -# the COPYRIGHT file. - -require 'spec_helper' -require 'rake_helpers' - -include RakeHelpers -describe RakeHelpers do - before do - @csv = Rails.root.join('spec', 'fixtures', 'test.csv') - end - - describe '#process_emails' do - before do - Devise.mailer.deliveries = [] - AppConfig.admins.account = FactoryGirl.create(:user).username - end - - it 'should send emails to each email' do - expect(EmailInviter).to receive(:new).exactly(3).times.and_return(double.as_null_object) - process_emails(@csv, 100, 1, false) - end - end -end - From f8bbe71f545c0c62d3e0efd38238ec824281a113 Mon Sep 17 00:00:00 2001 From: Steffen van Bergerem Date: Wed, 22 Jul 2015 00:59:06 +0200 Subject: [PATCH 2/2] Remove correlations closes #6223 --- Changelog.md | 1 + app/controllers/admins_controller.rb | 4 - app/views/admins/_admin_bar.haml | 1 - app/views/admins/correlations.haml | 12 -- config/initializers/load_libraries.rb | 1 - config/locales/diaspora/en.yml | 3 - config/routes.rb | 1 - lib/statistics.rb | 203 -------------------------- spec/lib/statistics_spec.rb | 177 ---------------------- 9 files changed, 1 insertion(+), 402 deletions(-) delete mode 100644 app/views/admins/correlations.haml delete mode 100644 lib/statistics.rb delete mode 100644 spec/lib/statistics_spec.rb diff --git a/Changelog.md b/Changelog.md index 816fea05b..f3525322f 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,6 +1,7 @@ # 0.5.3.0 ## Refactor +* Drop broken correlations from the admin pages [#6223](https://github.com/diaspora/diaspora/pull/6223) ## Bug fixes * Fix indentation and a link title on the default home page [#6212](https://github.com/diaspora/diaspora/pull/6212) diff --git a/app/controllers/admins_controller.rb b/app/controllers/admins_controller.rb index cf85c0184..f4b345ec7 100644 --- a/app/controllers/admins_controller.rb +++ b/app/controllers/admins_controller.rb @@ -75,10 +75,6 @@ class AdminsController < Admin::AdminController end - def correlations - @correlations_hash = Statistics.new.generate_correlations - end - private def percent_change(today, yesterday) diff --git a/app/views/admins/_admin_bar.haml b/app/views/admins/_admin_bar.haml index 75145d1be..a00e0c542 100644 --- a/app/views/admins/_admin_bar.haml +++ b/app/views/admins/_admin_bar.haml @@ -10,6 +10,5 @@ %li= link_to t('.weekly_user_stats'), weekly_user_stats_path %li= link_to t('.pod_stats'), pod_stats_path %li= link_to t('.report'), report_index_path - %li= link_to t('.correlations'), correlations_path %li= link_to t('.sidekiq_monitor'), sidekiq_path diff --git a/app/views/admins/correlations.haml b/app/views/admins/correlations.haml deleted file mode 100644 index 703f96af7..000000000 --- a/app/views/admins/correlations.haml +++ /dev/null @@ -1,12 +0,0 @@ -.container - %div - = render :partial => 'admins/admin_bar' - - %div.row - %div.span12 - %h1 - = t('.correlations_count') - %ul - - @correlations_hash.keys.each do |k| - %li - = "#{k.to_s}, #{@correlations_hash[k]}" diff --git a/config/initializers/load_libraries.rb b/config/initializers/load_libraries.rb index ff4f78492..f0dfd8c0c 100644 --- a/config/initializers/load_libraries.rb +++ b/config/initializers/load_libraries.rb @@ -20,7 +20,6 @@ require 'postzord' require 'publisher' require 'pubsubhubbub' require 'salmon' -require 'statistics' require 'stream' require 'webfinger' require 'webfinger_profile' diff --git a/config/locales/diaspora/en.yml b/config/locales/diaspora/en.yml index 7f9d9da6c..aaa452d7f 100644 --- a/config/locales/diaspora/en.yml +++ b/config/locales/diaspora/en.yml @@ -102,10 +102,7 @@ en: weekly_user_stats: "Weekly user stats" pod_stats: "Pod stats" report: "Reports" - correlations: "Correlations" sidekiq_monitor: "Sidekiq monitor" - correlations: - correlations_count: "Correlations with sign-in count:" user_search: you_currently: zero: "You currently have no invites left %{link}" diff --git a/config/routes.rb b/config/routes.rb index c0fd79d2b..db625cfcb 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -145,7 +145,6 @@ Diaspora::Application.routes.draw do match :user_search, via: [:get, :post] get :admin_inviter get :weekly_user_stats - get :correlations get :stats, :as => 'pod_stats' get "add_invites/:invite_code_id" => 'admins#add_invites', :as => 'add_invites' end diff --git a/lib/statistics.rb b/lib/statistics.rb deleted file mode 100644 index bc62695e9..000000000 --- a/lib/statistics.rb +++ /dev/null @@ -1,203 +0,0 @@ -class Statistics - - attr_reader :start_time, - :range - - def initialize - #@start_time = start_time - #@range = range - end - - def posts_count_sql - < 1 , "count" => 123}] - - x_array = [] - y_array = [] - - self.result_hash(first_metric).keys.each do |k| - if val = self.result_hash(second_metric)[k] - x_array << self.result_hash(first_metric)[k] - y_array << val - end - end - - correlation(x_array, y_array) - end - - def generate_correlations - result = {} - [:posts_count, :comments_count, :invites_sent_count, #:tags_followed_count, - :mentions_count, :contacts_sharing_with_count].each do |metric| - result[metric] = self.correlate(metric,:sign_in_count) - end - result - end - - def correlation(x_array, y_array) - sum = 0.0 - x_array.each_index do |i| - sum = sum + x_array[i].to_f * y_array[i].to_f - end - x_y_mean = sum/x_array.length.to_f - x_mean = mean(x_array) - y_mean = mean(y_array) - - st_dev_x = standard_deviation(x_array) - st_dev_y = standard_deviation(y_array) - - (x_y_mean - (x_mean*y_mean))/(st_dev_x * st_dev_y) - end - - def mean(array) - sum = array.inject(0.0) do |sum, val| - sum += val.to_f - end - sum / array.length - end - - def standard_deviation(array) - variance = lambda do - m = mean(array) - sum = 0.0 - array.each{ |v| sum += (v.to_f-m)**2 } - sum/array.length.to_f - end.call - - Math.sqrt(variance) - end - - ### % of cohort came back last week - def retention(n) - users_by_week(n).count.to_f/week_created(n).count - end - - def top_active_users(n) - ten_percent_lim = (users_by_week(n).count.to_f * 0.3).ceil - users_by_week(n).joins(:person => :profile).where('users.sign_in_count > 4').order("users.sign_in_count DESC").limit(ten_percent_lim).select('users.email, users.username, profiles.first_name, users.sign_in_count') - end - - def users_by_week(n) - week_created(n).where("current_sign_in_at > ?", Time.now - 1.week) - end - - protected - def where_clause_sql - if AppConfig.postgres? - "WHERE users.created_at > NOW() - '1 month'::INTERVAL" - else - "where users.created_at > FROM_UNIXTIME(#{(Time.now - 1.month).to_i})" - end - end - - def week_created(n) - User.where("username IS NOT NULL").where("users.created_at > ? and users.created_at < ?", Time.now - (n+1).weeks, Time.now - n.weeks) - end - - #@param [Symbol] input type - #@returns [Hash] of resulting query - def result_hash(type) - instance_hash = self.instance_variable_get("@#{type.to_s}_hash".to_sym) - unless instance_hash - post_count_array = User.connection.select_all(self.send("#{type.to_s}_sql".to_sym)) - - instance_hash = {} - post_count_array.each{ |h| instance_hash[h['id']] = h["count"]} - self.instance_variable_set("@#{type.to_s}_hash".to_sym, instance_hash) - end - instance_hash - end -end diff --git a/spec/lib/statistics_spec.rb b/spec/lib/statistics_spec.rb deleted file mode 100644 index 87597f062..000000000 --- a/spec/lib/statistics_spec.rb +++ /dev/null @@ -1,177 +0,0 @@ -require 'spec_helper' - -describe Statistics do - - def result_should_equal( actual ) - expect(actual.count).to eq(@result.count) - @result.each do |expected_hash| - expect(actual.find { |actual_hash| - actual_hash['id'].to_i == expected_hash['id'].to_i && - actual_hash['count'].to_i == expected_hash['count'].to_i - }).not_to be_nil - end - end - - before do - @time = Time.now - @stats = Statistics.new#(@time, @time - 1.week) - @result = [{"id" => alice.id , "count" => 0 }, - {"id" => bob.id , "count" => 1 }, - {"id" => eve.id , "count" => 0 }, - {"id" => local_luke.id , "count" => 0 }, - {"id" => local_leia.id , "count" => 0 }, - {"id" => peter.id , "count" => 0 }] - end - - describe '#posts_count_sql' do - it "pulls back an array of post counts and ids" do - FactoryGirl.create(:status_message, :author => bob.person) - result_should_equal User.connection.select_all(@stats.posts_count_sql) - end - end - - describe '#comments_count_sql' do - it "pulls back an array of post counts and ids" do - status_message = FactoryGirl.create(:status_message, :author => alice.person) - bob.comment!(status_message, "sup") - result_should_equal User.connection.select_all(@stats.comments_count_sql) - end - end - - - describe '#invites_sent_count_sql' do - it "pulls back an array of invite counts and ids" do - Invitation.batch_invite(["a@a.com"], :sender => bob, :aspect => bob.aspects.first, :service => 'email') - result_should_equal User.connection.select_all(@stats.invites_sent_count_sql) - end - end - - describe '#tags_followed_count_sql' do - it "pulls back an array of tag following counts and ids" do - TagFollowing.create!(:user => bob, :tag_id => 1) - result_should_equal User.connection.select_all(@stats.tags_followed_count_sql) - end - end - - describe '#mentions_count_sql' do - it "pulls back an array of mentions following counts and ids" do - post = FactoryGirl.create(:status_message, :author => bob.person) - Mention.create(:post => post, :person => bob.person) - result_should_equal User.connection.select_all(@stats.mentions_count_sql) - end - end - - describe '#contacts_sharing_with_count_sql' do - it "pulls back an array of mentions following counts and ids" do - # bob is sharing with alice and eve in the spec setup - alice.share_with(eve.person, alice.aspects.first) - @result = [{"id" => alice.id , "count" => 2 }, - {"id" => bob.id , "count" => 2 }, - {"id" => eve.id , "count" => 1 }, - {"id" => local_luke.id , "count" => 2 }, - {"id" => local_leia.id , "count" => 2 }, - {"id" => peter.id , "count" => 1 }] - - result_should_equal User.connection.select_all(@stats.contacts_sharing_with_count_sql) - end - end - - describe '#sign_in_count_sql' do - it "pulls back an array of sign_in_counts and ids" do - bob.sign_in_count = 1 - bob.save! - result_should_equal User.connection.select_all(@stats.sign_in_count_sql) - end - end - - describe "#fb_connected_distribution_sql" do - it "pulls back an array of sign_in_counts, connected, uids" do - bob.sign_in_count = 1 - bob.services << FactoryGirl.create(:service, :type => "Services::Facebook", :user => bob) - bob.save! - - eve.services << FactoryGirl.create(:service, :type => "Services::Facebook", :user => eve) - eve.save! - - - @result = [{"id" => alice.id , "count" => 0, "connected" => 0 }, - {"id" => bob.id , "count" => 1, "connected" => 1 }, - {"id" => eve.id , "count" => 0, "connected" => 1 }, - {"id" => local_luke.id , "count" => 0, "connected" => 0 }, - {"id" => local_leia.id , "count" => 0, "connected" => 0 }, - {"id" => peter.id , "count" => 0, "connected" => 0 }] - - expect(@stats.fb_connected_distribution).to match_array(@result) - end - end - - ["posts_count", "comments_count", "invites_sent_count", "tags_followed_count", - "mentions_count", "sign_in_count", "contacts_sharing_with_count" ].each do |method| - - it "#{method}_sql calls where_sql" do - expect(@stats).to receive(:where_clause_sql) - - @stats.send("#{method}_sql".to_sym) - end - - if !["sign_in_count", "tags_followed_count"].include?(method) - it "#generate_correlations calls correlate with #{method} and sign_in_count" do - allow(@stats).to receive(:correlate).and_return(0.5) - expect(@stats).to receive(:correlate).with(method.to_sym,:sign_in_count).and_return(0.75) - @stats.generate_correlations - end - end - end - - - describe "#correlation" do - it 'returns the correlation coefficient' do - expect(@stats.correlation([1,2],[1,2]).to_s).to eq(1.0.to_s) - expect(@stats.correlation([1,2,1,2],[1,1,2,2]).to_s).to eq(0.0.to_s) - end - end - describe "#generate_correlations" do - it 'returns the post count (and sign_in_count) correlation' do - bob.sign_in_count = 1 - bob.post(:status_message, :text => "here is a message") - bob.save! - - c = expect(@stats.generate_correlations[:posts_count].round(1)).to eq(1.0) - end - end - - describe "#correlate" do - it 'calls correlation with post' do - expect(User.connection).to receive(:select_all).and_return([{"id"=> 1, "count" => 7}, - {"id" => 2, "count" => 8}, - {"id" => 3, "count" => 9}], - [{"id"=> 1, "count" => 17}, - {"id" => 3, "count" => 19}] - ) - - expect(@stats).to receive(:correlation).with([7,9],[17,19]).and_return(0.5) - expect(@stats.correlate(:posts_count,:sign_in_count)).to eq(0.5) - end - end - - - - - context 'todos' do - before do - skip - end - - # requires a threshold - - describe '#disabled_email_count_sql' do - end - - # binary things - describe '#completed_getting_started_count_sql' do - end - - describe '.sign_up_method_sql' do - end - end -end