class Vanity::Experiment::BayesianBanditScore

Constants

DEFAULT_PROBABILITY

Public Class Methods

new(alternatives, outcome) click to toggle source
# File lib/vanity/experiment/bayesian_bandit_score.rb, line 10
def initialize(alternatives, outcome)
  @alternatives = alternatives
  @outcome = outcome
  @method = :bayes_bandit_score
end

Public Instance Methods

calculate!(probability=DEFAULT_PROBABILITY) click to toggle source
# File lib/vanity/experiment/bayesian_bandit_score.rb, line 16
def calculate!(probability=DEFAULT_PROBABILITY)
  # sort by conversion rate to find second best
  @alts = @alternatives.sort_by(&:measure)
  @base = @alts[-2]

  assign_alternatives_bayesian_probability(@alts)

  @least = assign_alternatives_difference(@alts)

  # best alternative is one with highest conversion rate (best shot).
  @best = @alts.last if @alts.last.measure > 0.0
  # choice alternative can only pick best if we have high probability (>90%).
  @choice = outcome_or_best_probability(@alternatives, @outcome, @best, probability)
  self
end

Protected Instance Methods

assign_alternatives_bayesian_probability(alternatives) click to toggle source

Assign each alternative's bayesian probability of being the best alternative to alternative#probability.

# File lib/vanity/experiment/bayesian_bandit_score.rb, line 46
def assign_alternatives_bayesian_probability(alternatives)
  alternative_posteriors = calculate_alternative_posteriors(alternatives)
  alternatives.each_with_index do |alternative, i|
    alternative.probability = 100 * probability_alternative_is_best(alternative_posteriors[i], alternative_posteriors)
  end
end
assign_alternatives_difference(alternatives) click to toggle source
# File lib/vanity/experiment/bayesian_bandit_score.rb, line 79
def assign_alternatives_difference(alternatives)
  # difference is measured from least performant
  least = alternatives.find { |alternative| alternative.measure > 0 }
  if least
    alternatives.each do |alternative|
      if alternative.measure > least.measure
        alternative.difference = (alternative.measure - least.measure) / least.measure * 100
      end
    end
  end
  least
end
calculate_alternative_posteriors(alternatives) click to toggle source
# File lib/vanity/experiment/bayesian_bandit_score.rb, line 53
def calculate_alternative_posteriors(alternatives)
  alternatives.map do |alternative|
    x = alternative.converted
    n = alternative.participants
    Rubystats::BetaDistribution.new(x+1, n-x+1)
  end
end
outcome_or_best_probability(alternatives, outcome, best, probability) click to toggle source
# File lib/vanity/experiment/bayesian_bandit_score.rb, line 34
def outcome_or_best_probability(alternatives, outcome, best, probability)
  if outcome
    alternatives[@outcome.id]
  elsif best && best.probability >= probability
    best
  else
    nil
  end
end
pdf_alternative_is_best(z, alternative_being_examined, all_alternatives) click to toggle source
# File lib/vanity/experiment/bayesian_bandit_score.rb, line 67
def pdf_alternative_is_best(z, alternative_being_examined, all_alternatives)
  # get the pdf for this alternative at z
  pdf = alternative_being_examined.pdf(z)
  # now multiply by the probability that all the other alternatives are lower
  all_alternatives.each do |alternative|
    if alternative != alternative_being_examined
      pdf = pdf * alternative.cdf(z)
    end
  end
  pdf
end
probability_alternative_is_best(alternative_being_examined, all_alternatives) click to toggle source
# File lib/vanity/experiment/bayesian_bandit_score.rb, line 61
def probability_alternative_is_best(alternative_being_examined, all_alternatives)
  Integration.integrate(0, 1, :tolerance=>1e-4) do |z|
    pdf_alternative_is_best(z, alternative_being_examined, all_alternatives)
  end
end