class DerailedBenchmarks::StatsFromDir
A class used to read several benchmark files it will parse each file, then sort by average time of benchmarks. It can be used to find the fastest and slowest examples and give information about them such as what the percent difference is and if the results are statistically significant
Example:
branch_info = {} branch_info["loser"] = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" } branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" } stats = DerailedBenchmarks::StatsFromDir.new(branch_info) stats.newest.average # => 10.5 stats.oldest.average # => 11.0 stats.significant? # => true stats.x_faster # => "1.0476"
Constants
- FORMAT
Attributes
newest[R]
oldest[R]
stats[R]
Public Class Methods
new(input)
click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 32 def initialize(input) @files = [] if input.is_a?(Hash) hash = input hash.each do |branch, info_hash| file = info_hash.fetch(:file) desc = info_hash.fetch(:desc) time = info_hash.fetch(:time) short_sha = info_hash[:short_sha] @files << StatsForFile.new(file: file, desc: desc, time: time, name: branch, short_sha: short_sha) end else input.each do |commit| @files << StatsForFile.new( file: commit.file, desc: commit.desc, time: commit.time, name: commit.ref, short_sha: commit.short_sha ) end end @files.sort_by! { |f| f.time } @oldest = @files.first @newest = @files.last end
Public Instance Methods
align()
click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 118 def align " " * (percent_faster.to_s.index(".") - x_faster.to_s.index(".")) end
call()
click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 60 def call @files.each(&:call) return self if @files.detect(&:empty?) stats_95 = statistical_test(confidence: 95) # If default check is good, see if we also pass a more rigorous test # if so, then use the more rigourous test if stats_95[:alternative] stats_99 = statistical_test(confidence: 99) @stats = stats_99 if stats_99[:alternative] end @stats ||= stats_95 self end
change_direction()
click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 110 def change_direction if faster? "FASTER 🚀🚀🚀" else "SLOWER 🐢🐢🐢" end end
d_critical()
click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 94 def d_critical @stats[:d_critical].to_f end
d_max()
click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 90 def d_max @stats[:d_max].to_f end
faster?()
click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 102 def faster? newest.median < oldest.median end
histogram(io = $stdout)
click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 122 def histogram(io = $stdout) dual_histogram = MiniHistogram.dual_plot do |a, b| a.values = newest.values a.options = { title: "\n [#{newest.short_sha || newest.name}] description:\n #{newest.desc.inspect}", xlabel: "# of runs in range" } b.values = oldest.values b.options = { title: "\n [#{oldest.short_sha || oldest.name}] description:\n #{oldest.desc.inspect}", xlabel: "# of runs in range" } end io.puts io.puts "Histograms (time ranges are in seconds):" io.puts(dual_histogram) io.puts end
percent_faster()
click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 106 def percent_faster (((oldest.median - newest.median) / oldest.median).to_f * 100) end
significant?()
click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 86 def significant? @stats[:alternative] end
statistical_test(series_1=oldest.values, series_2=newest.values, confidence: 95)
click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 78 def statistical_test(series_1=oldest.values, series_2=newest.values, confidence: 95) StatisticalTest::KSTest.two_samples( group_one: series_1, group_two: series_2, alpha: (100 - confidence) / 100.0 ) end
x_faster()
click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 98 def x_faster (oldest.median/newest.median).to_f end