class DerailedBenchmarks::StatsFromDir

A class used to read several benchmark files it will parse each file, then sort by average time of benchmarks. It can be used to find the fastest and slowest examples and give information about them such as what the percent difference is and if the results are statistically significant

Example:

branch_info = {}
branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
stats = DerailedBenchmarks::StatsFromDir.new(branch_info)

stats.newest.average  # => 10.5
stats.oldest.average  # => 11.0
stats.significant?    # => true
stats.x_faster        # => "1.0476"

Constants

FORMAT

Attributes

newest[R]
oldest[R]
stats[R]

Public Class Methods

new(input) click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 32
def initialize(input)
  @files = []

  if input.is_a?(Hash)
    hash = input
    hash.each do |branch, info_hash|
      file = info_hash.fetch(:file)
      desc = info_hash.fetch(:desc)
      time = info_hash.fetch(:time)
      short_sha = info_hash[:short_sha]
      @files << StatsForFile.new(file: file, desc: desc, time: time, name: branch, short_sha: short_sha)
    end
  else
    input.each do |commit|
      @files << StatsForFile.new(
        file: commit.file,
        desc: commit.desc,
        time: commit.time,
        name: commit.ref,
        short_sha: commit.short_sha
      )
    end
  end
  @files.sort_by! { |f| f.time }
  @oldest = @files.first
  @newest = @files.last
end

Public Instance Methods

align() click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 118
def align
  " " * (percent_faster.to_s.index(".") - x_faster.to_s.index("."))
end
banner(io = $stdout) click to toggle source
call() click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 60
def call
  @files.each(&:call)

  return self if @files.detect(&:empty?)

  stats_95 = statistical_test(confidence: 95)

  # If default check is good, see if we also pass a more rigorous test
  # if so, then use the more rigourous test
  if stats_95[:alternative]
    stats_99 = statistical_test(confidence: 99)
    @stats = stats_99 if stats_99[:alternative]
  end
  @stats ||= stats_95

  self
end
change_direction() click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 110
def change_direction
  if faster?
    "FASTER 🚀🚀🚀"
  else
    "SLOWER 🐢🐢🐢"
  end
end
d_critical() click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 94
def d_critical
  @stats[:d_critical].to_f
end
d_max() click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 90
def d_max
  @stats[:d_max].to_f
end
faster?() click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 102
def faster?
  newest.median < oldest.median
end
histogram(io = $stdout) click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 122
def histogram(io = $stdout)
  dual_histogram = MiniHistogram.dual_plot do |a, b|
    a.values = newest.values
    a.options = {
      title: "\n   [#{newest.short_sha || newest.name}] description:\n     #{newest.desc.inspect}",
      xlabel: "# of runs in range"
    }
    b.values = oldest.values
    b.options = {
      title: "\n   [#{oldest.short_sha || oldest.name}] description:\n     #{oldest.desc.inspect}",
      xlabel: "# of runs in range"
    }
  end

  io.puts
  io.puts "Histograms (time ranges are in seconds):"
  io.puts(dual_histogram)
  io.puts
end
percent_faster() click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 106
def percent_faster
  (((oldest.median - newest.median) / oldest.median).to_f  * 100)
end
significant?() click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 86
def significant?
  @stats[:alternative]
end
statistical_test(series_1=oldest.values, series_2=newest.values, confidence: 95) click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 78
def statistical_test(series_1=oldest.values, series_2=newest.values, confidence: 95)
  StatisticalTest::KSTest.two_samples(
    group_one: series_1,
    group_two: series_2,
    alpha: (100 - confidence) / 100.0
  )
end
x_faster() click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 98
def x_faster
  (oldest.median/newest.median).to_f
end