module Abongo::Statistics
Constants
- DESCRIPTION_IN_WORDS
- HANDY_Z_SCORE_CHEATSHEET
- PERCENTAGES
Public Class Methods
conversion_rate(exp)
click to toggle source
# File lib/abongo/statistics.rb, line 50 def self.conversion_rate(exp) 1.0 * exp['conversions'] / exp['participants'] end
describe_result_in_words(experiment, alternatives)
click to toggle source
# File lib/abongo/statistics.rb, line 58 def self.describe_result_in_words(experiment, alternatives) begin z = zscore(alternatives) rescue return "Could not execute the significance test because one or more of the alternatives has not been seen yet." end p = p_value(alternatives) words = "" if (alternatives[0]['participants'] < 10) || (alternatives[1]['participants'] < 10) words += "Take these results with a grain of salt since your samples are so small: " end best_alternative = alternatives.max{|a, b| conversion_rate(a) <=> conversion_rate(b)} alts = alternatives - [best_alternative] worst_alternative = alts.first words += "The best alternative you have is: [#{best_alternative['content']}], which had " words += "#{best_alternative['conversions']} conversions from #{best_alternative['participants']} participants " words += "(#{pretty_conversion_rate(best_alternative)}). The other alternative was [#{worst_alternative['content']}], " words += "which had #{worst_alternative['conversions']} conversions from #{worst_alternative['participants']} participants " words += "(#{pretty_conversion_rate(worst_alternative)}). " if (p.nil?) words += "However, this difference is not statistically significant." else words += "This difference is #{PERCENTAGES[p]} likely to be statistically significant, which means you can be " words += "#{DESCRIPTION_IN_WORDS[p]} that it is the result of your alternatives actually mattering, rather than " words += "being due to random chance. However, this statistical test can't measure how likely the currently " words += "observed magnitude of the difference is to be accurate or not. It only says \"better\", not \"better " words += "by so much\"." end words end
is_statistically_significant?(p = 0.05)
click to toggle source
# File lib/abongo/statistics.rb, line 46 def self.is_statistically_significant?(p = 0.05) p_value <= p end
p_value(alternatives)
click to toggle source
# File lib/abongo/statistics.rb, line 32 def self.p_value(alternatives) index = 0 z = zscore(alternatives) z = z.abs found_p = nil while index < HANDY_Z_SCORE_CHEATSHEET.size do if (z > HANDY_Z_SCORE_CHEATSHEET[index][1]) found_p = HANDY_Z_SCORE_CHEATSHEET[index][0] end index += 1 end found_p end
pretty_conversion_rate(exp)
click to toggle source
# File lib/abongo/statistics.rb, line 54 def self.pretty_conversion_rate(exp) sprintf("%4.2f%%", conversion_rate(exp) * 100) end
zscore(alternatives)
click to toggle source
# File lib/abongo/statistics.rb, line 10 def self.zscore(alternatives) if alternatives.size != 2 raise "Sorry, can't currently automatically calculate statistics for A/B tests with > 2 alternatives." end if (alternatives[0]['participants'] == 0) || (alternatives[1]['participants'] == 0) raise "Can't calculate the z score if either of the alternatives lacks participants." end cr1 = conversion_rate(alternatives[0]) cr2 = conversion_rate(alternatives[1]) n1 = alternatives[0]['participants'] n2 = alternatives[1]['participants'] numerator = cr1 - cr2 frac1 = cr1 * (1 - cr1) / n1 frac2 = cr2 * (1 - cr2) / n2 numerator / ((frac1 + frac2) ** 0.5) end