class ABProf::BenchmarkInstance
Attributes
reports[R]
Public Class Methods
new()
click to toggle source
# File lib/abprof/benchmark_dsl.rb, line 16 def initialize @pvalue = 0.05 @burnin = 10 @min_trials = 1 @max_trials = 20 @iters_per_trial = 10 @bare = false @static_order = false @state = { :samples => [[], []], :p_tests => [], :iter => 0, } end
Public Instance Methods
print_summary(opts = {})
click to toggle source
# File lib/abprof/benchmark_dsl.rb, line 116 def print_summary(opts = {}) p_val = @state[:p_tests][-1] diverged = false if p_val < @pvalue puts "Based on measured P value #{p_val}, we believe there is a speed difference." puts "As of end of run, p value is #{p_val}. Now run more times to check, or with lower p." summary11 = ABProf.summarize("mean", @state[:samples][0]) summary12 = ABProf.summarize("median", @state[:samples][0]) summary21 = ABProf.summarize("mean", @state[:samples][1]) summary22 = ABProf.summarize("median", @state[:samples][1]) fastest = "1" command = @reports[0] mean_times = summary21 / summary11 median_times = summary22 / summary12 if summary12 > summary22 fastest = "2" command = @reports[1] mean_times = summary11 / summary21 median_times = summary12 / summary22 end puts "Lower (faster?) process is #{fastest}, command line: #{command.inspect}" puts "Lower command is (very) roughly #{median_times} times lower (faster?) -- assuming linear sampling, checking at median." puts " Checking at mean, it would be #{mean_times} lower (faster?)." print "\n" puts "Process 1 mean result: #{summary11}" puts "Process 1 median result: #{summary12}" puts "Process 2 mean result: #{summary21}" puts "Process 2 median result: #{summary22}" else puts "Based on measured P value #{p_val} and threshold #{@pvalue}, we believe there is" puts "no significant difference detectable with this set of trials." puts "If you believe there is a small difference that wasn't detected, try raising the number" puts "of iterations per trial, or the maximum number of trials." diverged = true end end
report(&block)
click to toggle source
# File lib/abprof/benchmark_dsl.rb, line 41 def report(&block) @reports ||= [] @reports.push block end
report_command(cmd)
click to toggle source
# File lib/abprof/benchmark_dsl.rb, line 36 def report_command(cmd) @reports ||= [] @reports.push cmd end
run_burnin(opts = {})
click to toggle source
# File lib/abprof/benchmark_dsl.rb, line 46 def run_burnin(opts = {}) return unless @burnin > 0 @process1.run_iters @burnin @process2.run_iters @burnin end
run_one_trial(pts = {})
click to toggle source
# File lib/abprof/benchmark_dsl.rb, line 53 def run_one_trial(pts = {}) order_rand = (rand() * 2.0).to_i if @static_order || order_rand == 0 @state[:samples][0] += @process1.run_iters @iters_per_trial @state[:samples][1] += @process2.run_iters @iters_per_trial else # Same thing, but do process2 first @state[:samples][1] += @process2.run_iters @iters_per_trial @state[:samples][0] += @process1.run_iters @iters_per_trial end @state[:iter] += 1 end
run_sampling(opts = {})
click to toggle source
# File lib/abprof/benchmark_dsl.rb, line 66 def run_sampling(opts = {}) process_type = @bare ? ABProf::ABBareProcess : ABProf::ABHarnessProcess command1 = @reports[0] command2 = @reports[1] @process1 = process_type.new command1, :debug => @debug @process2 = process_type.new command2, :debug => @debug run_burnin opts puts "Beginning sampling from processes." if opts[:print_output] # Sampling p_val = 1.0 @max_trials.times do run_one_trial opts # No t-test without 3+ samples if @state[:samples][0].size > 2 # Evaluate the Welch's t-test t = Statsample::Test.t_two_samples_independent(@state[:samples][0].to_vector, @state[:samples][1].to_vector) p_val = t.probability_not_equal_variance @state[:p_tests].push p_val avg_1 = @state[:samples][0].inject(0.0, &:+) / @state[:samples][0].length avg_2 = @state[:samples][1].inject(0.0, &:+) / @state[:samples][1].length smaller = "1" smaller = "2" if avg_1 > avg_2 puts "Trial #{@state[:iter]}, Welch's T-test p value: #{p_val.inspect} (Guessed smaller: #{smaller})" if opts[:print_output] end # Just finished trial number i+1. So we can exit only if i+1 was at least # the minimum number of trials. break if p_val < @pvalue && (@state[:iter] + 1 >= @min_trials) end # Clean up processes @process1.kill @process2.kill print_summary opts unless opts[:no_print_summary] p_val = @state[:p_tests][-1] if p_val >= @pvalue && @fail_on_divergence STDERR.puts "Measured P value of #{p_val.inspect} is higher than allowable P value of #{@pvalue.inspect}!" exit 2 end @state end