nocebo/diaspora
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove correlations

Browse source 
closes #6223
This commit is contained in:
Steffen van Bergerem 2015-07-22 00:59:06 +02:00 committed by Jonne Haß
parent c8f482064c
commit f8bbe71f54
9 changed files with 1 additions and 402 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Changelog.md
View file

@ -1,6 +1,7 @@
# 0.5.3.0
## Refactor
* Drop broken correlations from the admin pages [#6223](https://github.com/diaspora/diaspora/pull/6223)
## Bug fixes
* Fix indentation and a link title on the default home page [#6212](https://github.com/diaspora/diaspora/pull/6212)

4
app/controllers/admins_controller.rb
View file

@ -75,10 +75,6 @@ class AdminsController < Admin::AdminController
end
def correlations
@correlations_hash = Statistics.new.generate_correlations
end
private
def percent_change(today, yesterday)

1
app/views/admins/_admin_bar.haml
View file

@ -10,6 +10,5 @@
%li= link_to t('.weekly_user_stats'), weekly_user_stats_path
%li= link_to t('.pod_stats'), pod_stats_path
%li= link_to t('.report'), report_index_path
%li= link_to t('.correlations'), correlations_path
%li= link_to t('.sidekiq_monitor'), sidekiq_path

12
app/views/admins/correlations.haml
View file

@ -1,12 +0,0 @@
.container
%div
= render :partial => 'admins/admin_bar'
%div.row
%div.span12
%h1
= t('.correlations_count')
%ul
- @correlations_hash.keys.each do |k|
%li
= "#{k.to_s}, #{@correlations_hash[k]}"

1
config/initializers/load_libraries.rb
View file

@ -20,7 +20,6 @@ require 'postzord'
require 'publisher'
require 'pubsubhubbub'
require 'salmon'
require 'statistics'
require 'stream'
require 'webfinger'
require 'webfinger_profile'

3
config/locales/diaspora/en.yml
View file

@ -102,10 +102,7 @@ en:
weekly_user_stats: "Weekly user stats"
pod_stats: "Pod stats"
report: "Reports"
correlations: "Correlations"
sidekiq_monitor: "Sidekiq monitor"
correlations:
correlations_count: "Correlations with sign-in count:"
user_search:
you_currently:
zero: "You currently have no invites left %{link}"

1
config/routes.rb
View file

@ -145,7 +145,6 @@ Diaspora::Application.routes.draw do
match :user_search, via: [:get, :post]
get :admin_inviter
get :weekly_user_stats
get :correlations
get :stats, :as => 'pod_stats'
get "add_invites/:invite_code_id" => 'admins#add_invites', :as => 'add_invites'
end

203
lib/statistics.rb
View file

@ -1,203 +0,0 @@
class Statistics
attr_reader :start_time,
:range
def initialize
#@start_time = start_time
#@range = range
end
def posts_count_sql
<<SQL
SELECT users.id AS id, count(posts.id) AS count
FROM users
JOIN people ON people.owner_id = users.id
LEFT OUTER JOIN posts ON people.id = posts.author_id
#{self.where_clause_sql}
GROUP BY users.id
SQL
end
def comments_count_sql
<<SQL
SELECT users.id AS id, count(comments.id) AS count
FROM users
JOIN people ON people.owner_id = users.id
LEFT OUTER JOIN comments ON people.id = comments.author_id
#{self.where_clause_sql}
GROUP BY users.id
SQL
end
def invites_sent_count_sql
<<SQL
SELECT users.id AS id, count(invitations.id) AS count
FROM users
LEFT OUTER JOIN invitations ON users.id = invitations.sender_id
#{self.where_clause_sql}
GROUP BY users.id
SQL
end
def tags_followed_count_sql
<<SQL
SELECT users.id AS id, count(tag_followings.id) AS count
FROM users
LEFT OUTER JOIN tag_followings on users.id = tag_followings.user_id
#{self.where_clause_sql}
GROUP BY users.id
SQL
end
def mentions_count_sql
<<SQL
SELECT users.id AS id, count(mentions.id) AS count
FROM users
JOIN people on users.id = people.owner_id
LEFT OUTER JOIN mentions on people.id = mentions.person_id
#{self.where_clause_sql}
GROUP BY users.id
SQL
end
def contacts_sharing_with_count_sql
<<SQL
SELECT users.id AS id, count(contacts.id) AS count
FROM users
JOIN contacts on contacts.user_id = users.id
JOIN aspect_memberships on aspect_memberships.contact_id = contacts.id
#{self.where_clause_sql}
GROUP BY users.id
SQL
end
def fb_connected_distribution_sql
<<SQL
SELECT users.id AS id, users.sign_in_count AS count, count(services.id) AS connected
FROM users
LEFT OUTER JOIN services on services.user_id = users.id
AND services.type = 'Services::Facebook'
#{self.where_clause_sql}
GROUP BY users.id, users.sign_in_count
SQL
end
def fb_connected_distribution
User.connection.select_all(fb_connected_distribution_sql).map { |row|
Hash[
row.map { |k,v|
[k, v.to_i]
}
]
}
end
def sign_in_count_sql
<<SQL
SELECT users.id AS id, users.sign_in_count AS count
FROM users
#{self.where_clause_sql}
SQL
end
def correlate(first_metric, second_metric)
# [{"id" => 1 , "count" => 123}]
x_array = []
y_array = []
self.result_hash(first_metric).keys.each do |k|
if val = self.result_hash(second_metric)[k]
x_array << self.result_hash(first_metric)[k]
y_array << val
end
end
correlation(x_array, y_array)
end
def generate_correlations
result = {}
[:posts_count, :comments_count, :invites_sent_count, #:tags_followed_count,
:mentions_count, :contacts_sharing_with_count].each do |metric|
result[metric] = self.correlate(metric,:sign_in_count)
end
result
end
def correlation(x_array, y_array)
sum = 0.0
x_array.each_index do |i|
sum = sum + x_array[i].to_f * y_array[i].to_f
end
x_y_mean = sum/x_array.length.to_f
x_mean = mean(x_array)
y_mean = mean(y_array)
st_dev_x = standard_deviation(x_array)
st_dev_y = standard_deviation(y_array)
(x_y_mean - (x_mean*y_mean))/(st_dev_x * st_dev_y)
end
def mean(array)
sum = array.inject(0.0) do |sum, val|
sum += val.to_f
end
sum / array.length
end
def standard_deviation(array)
variance = lambda do
m = mean(array)
sum = 0.0
array.each{ |v| sum += (v.to_f-m)**2 }
sum/array.length.to_f
end.call
Math.sqrt(variance)
end
### % of cohort came back last week
def retention(n)
users_by_week(n).count.to_f/week_created(n).count
end
def top_active_users(n)
ten_percent_lim = (users_by_week(n).count.to_f * 0.3).ceil
users_by_week(n).joins(:person => :profile).where('users.sign_in_count > 4').order("users.sign_in_count DESC").limit(ten_percent_lim).select('users.email, users.username, profiles.first_name, users.sign_in_count')
end
def users_by_week(n)
week_created(n).where("current_sign_in_at > ?", Time.now - 1.week)
end
protected
def where_clause_sql
if AppConfig.postgres?
"WHERE users.created_at > NOW() - '1 month'::INTERVAL"
else
"where users.created_at > FROM_UNIXTIME(#{(Time.now - 1.month).to_i})"
end
end
def week_created(n)
User.where("username IS NOT NULL").where("users.created_at > ? and users.created_at < ?", Time.now - (n+1).weeks, Time.now - n.weeks)
end
#@param [Symbol] input type
#@returns [Hash] of resulting query
def result_hash(type)
instance_hash = self.instance_variable_get("@#{type.to_s}_hash".to_sym)
unless instance_hash
post_count_array = User.connection.select_all(self.send("#{type.to_s}_sql".to_sym))
instance_hash = {}
post_count_array.each{ |h| instance_hash[h['id']] = h["count"]}
self.instance_variable_set("@#{type.to_s}_hash".to_sym, instance_hash)
end
instance_hash
end
end

177
spec/lib/statistics_spec.rb
View file

@ -1,177 +0,0 @@
require 'spec_helper'
describe Statistics do
def result_should_equal( actual )
expect(actual.count).to eq(@result.count)
@result.each do |expected_hash|
expect(actual.find { |actual_hash|
actual_hash['id'].to_i == expected_hash['id'].to_i &&
actual_hash['count'].to_i == expected_hash['count'].to_i
}).not_to be_nil
end
end
before do
@time = Time.now
@stats = Statistics.new#(@time, @time - 1.week)
@result = [{"id" => alice.id , "count" => 0 },
{"id" => bob.id , "count" => 1 },
{"id" => eve.id , "count" => 0 },
{"id" => local_luke.id , "count" => 0 },
{"id" => local_leia.id , "count" => 0 },
{"id" => peter.id , "count" => 0 }]
end
describe '#posts_count_sql' do
it "pulls back an array of post counts and ids" do
FactoryGirl.create(:status_message, :author => bob.person)
result_should_equal User.connection.select_all(@stats.posts_count_sql)
end
end
describe '#comments_count_sql' do
it "pulls back an array of post counts and ids" do
status_message = FactoryGirl.create(:status_message, :author => alice.person)
bob.comment!(status_message, "sup")
result_should_equal User.connection.select_all(@stats.comments_count_sql)
end
end
describe '#invites_sent_count_sql' do
it "pulls back an array of invite counts and ids" do
Invitation.batch_invite(["a@a.com"], :sender => bob, :aspect => bob.aspects.first, :service => 'email')
result_should_equal User.connection.select_all(@stats.invites_sent_count_sql)
end
end
describe '#tags_followed_count_sql' do
it "pulls back an array of tag following counts and ids" do
TagFollowing.create!(:user => bob, :tag_id => 1)
result_should_equal User.connection.select_all(@stats.tags_followed_count_sql)
end
end
describe '#mentions_count_sql' do
it "pulls back an array of mentions following counts and ids" do
post = FactoryGirl.create(:status_message, :author => bob.person)
Mention.create(:post => post, :person => bob.person)
result_should_equal User.connection.select_all(@stats.mentions_count_sql)
end
end
describe '#contacts_sharing_with_count_sql' do
it "pulls back an array of mentions following counts and ids" do
# bob is sharing with alice and eve in the spec setup
alice.share_with(eve.person, alice.aspects.first)
@result = [{"id" => alice.id , "count" => 2 },
{"id" => bob.id , "count" => 2 },
{"id" => eve.id , "count" => 1 },
{"id" => local_luke.id , "count" => 2 },
{"id" => local_leia.id , "count" => 2 },
{"id" => peter.id , "count" => 1 }]
result_should_equal User.connection.select_all(@stats.contacts_sharing_with_count_sql)
end
end
describe '#sign_in_count_sql' do
it "pulls back an array of sign_in_counts and ids" do
bob.sign_in_count = 1
bob.save!
result_should_equal User.connection.select_all(@stats.sign_in_count_sql)
end
end
describe "#fb_connected_distribution_sql" do
it "pulls back an array of sign_in_counts, connected, uids" do
bob.sign_in_count = 1
bob.services << FactoryGirl.create(:service, :type => "Services::Facebook", :user => bob)
bob.save!
eve.services << FactoryGirl.create(:service, :type => "Services::Facebook", :user => eve)
eve.save!
@result = [{"id" => alice.id , "count" => 0, "connected" => 0 },
{"id" => bob.id , "count" => 1, "connected" => 1 },
{"id" => eve.id , "count" => 0, "connected" => 1 },
{"id" => local_luke.id , "count" => 0, "connected" => 0 },
{"id" => local_leia.id , "count" => 0, "connected" => 0 },
{"id" => peter.id , "count" => 0, "connected" => 0 }]
expect(@stats.fb_connected_distribution).to match_array(@result)
end
end
["posts_count", "comments_count", "invites_sent_count", "tags_followed_count",
"mentions_count", "sign_in_count", "contacts_sharing_with_count" ].each do |method|
it "#{method}_sql calls where_sql" do
expect(@stats).to receive(:where_clause_sql)
@stats.send("#{method}_sql".to_sym)
end
if !["sign_in_count", "tags_followed_count"].include?(method)
it "#generate_correlations calls correlate with #{method} and sign_in_count" do
allow(@stats).to receive(:correlate).and_return(0.5)
expect(@stats).to receive(:correlate).with(method.to_sym,:sign_in_count).and_return(0.75)
@stats.generate_correlations
end
end
end
describe "#correlation" do
it 'returns the correlation coefficient' do
expect(@stats.correlation([1,2],[1,2]).to_s).to eq(1.0.to_s)
expect(@stats.correlation([1,2,1,2],[1,1,2,2]).to_s).to eq(0.0.to_s)
end
end
describe "#generate_correlations" do
it 'returns the post count (and sign_in_count) correlation' do
bob.sign_in_count = 1
bob.post(:status_message, :text => "here is a message")
bob.save!
c = expect(@stats.generate_correlations[:posts_count].round(1)).to eq(1.0)
end
end
describe "#correlate" do
it 'calls correlation with post' do
expect(User.connection).to receive(:select_all).and_return([{"id"=> 1, "count" => 7},
{"id" => 2, "count" => 8},
{"id" => 3, "count" => 9}],
[{"id"=> 1, "count" => 17},
{"id" => 3, "count" => 19}]
)
expect(@stats).to receive(:correlation).with([7,9],[17,19]).and_return(0.5)
expect(@stats.correlate(:posts_count,:sign_in_count)).to eq(0.5)
end
end
context 'todos' do
before do
skip
end
# requires a threshold
describe '#disabled_email_count_sql' do
end
# binary things
describe '#completed_getting_started_count_sql' do
end
describe '.sign_up_method_sql' do
end
end
end