class ABProf::BenchmarkInstance

Attributes

reports[R]

Public Class Methods

new() click to toggle source
# File lib/abprof/benchmark_dsl.rb, line 16
def initialize
  @pvalue = 0.05
  @burnin = 10
  @min_trials = 1
  @max_trials = 20
  @iters_per_trial = 10
  @bare = false
  @static_order = false

  @state = {
    :samples => [[], []],
    :p_tests => [],
    :iter => 0,
  }
end

Public Instance Methods

print_summary(opts = {}) click to toggle source
report(&block) click to toggle source
# File lib/abprof/benchmark_dsl.rb, line 41
def report(&block)
  @reports ||= []
  @reports.push block
end
report_command(cmd) click to toggle source
# File lib/abprof/benchmark_dsl.rb, line 36
def report_command(cmd)
  @reports ||= []
  @reports.push cmd
end
run_burnin(opts = {}) click to toggle source
# File lib/abprof/benchmark_dsl.rb, line 46
def run_burnin(opts = {})
  return unless @burnin > 0

  @process1.run_iters @burnin
  @process2.run_iters @burnin
end
run_one_trial(pts = {}) click to toggle source
# File lib/abprof/benchmark_dsl.rb, line 53
def run_one_trial(pts = {})
  order_rand = (rand() * 2.0).to_i
  if @static_order || order_rand == 0
    @state[:samples][0] += @process1.run_iters @iters_per_trial
    @state[:samples][1] += @process2.run_iters @iters_per_trial
  else
    # Same thing, but do process2 first
    @state[:samples][1] += @process2.run_iters @iters_per_trial
    @state[:samples][0] += @process1.run_iters @iters_per_trial
  end
  @state[:iter] += 1
end
run_sampling(opts = {}) click to toggle source
# File lib/abprof/benchmark_dsl.rb, line 66
def run_sampling(opts = {})
  process_type = @bare ? ABProf::ABBareProcess : ABProf::ABHarnessProcess
  command1 = @reports[0]
  command2 = @reports[1]

  @process1 = process_type.new command1, :debug => @debug
  @process2 = process_type.new command2, :debug => @debug

  run_burnin opts

  puts "Beginning sampling from processes." if opts[:print_output]

  # Sampling
  p_val = 1.0
  @max_trials.times do
    run_one_trial opts

    # No t-test without 3+ samples
    if @state[:samples][0].size > 2
      # Evaluate the Welch's t-test
      t = Statsample::Test.t_two_samples_independent(@state[:samples][0].to_vector, @state[:samples][1].to_vector)
      p_val = t.probability_not_equal_variance
      @state[:p_tests].push p_val
      avg_1 = @state[:samples][0].inject(0.0, &:+) / @state[:samples][0].length
      avg_2 = @state[:samples][1].inject(0.0, &:+) / @state[:samples][1].length
      smaller = "1"
      smaller = "2" if avg_1 > avg_2
      puts "Trial #{@state[:iter]}, Welch's T-test p value: #{p_val.inspect}   (Guessed smaller: #{smaller})" if opts[:print_output]
    end

    # Just finished trial number i+1. So we can exit only if i+1 was at least
    # the minimum number of trials.
    break if p_val < @pvalue && (@state[:iter] + 1 >= @min_trials)
  end

  # Clean up processes
  @process1.kill
  @process2.kill

  print_summary opts unless opts[:no_print_summary]

  p_val = @state[:p_tests][-1]
  if p_val >= @pvalue && @fail_on_divergence
    STDERR.puts "Measured P value of #{p_val.inspect} is higher than allowable P value of #{@pvalue.inspect}!"
    exit 2
  end

  @state
end