class Benchmark::IPS::Job

Benchmark jobs.

Benchmark jobs.

Constants

MAX_TIME_SKEW

The percentage of the expected runtime to allow before reporting a weird runtime

MICROSECONDS_PER_100MS

Microseconds per 100 millisecond.

MICROSECONDS_PER_SECOND

Microseconds per second.

POW_2_30

Attributes

compare[R]

Determining whether to run comparison utility. @return [Boolean] true if needs to run compare.

confidence[RW]

Confidence. @return [Integer]

full_report[R]

Report object containing information about the run. @return [Report] the report object.

hold[RW]

Determining whether to hold results between Ruby invocations @return [Boolean]

iterations[RW]

Warmup and calculation iterations. @return [Integer]

list[R]

Two-element arrays, consisting of label and block pairs. @return [Array<Entry>] list of entries

quiet[R]

Silence output @return [Boolean]

stats[RW]

Statistics model. @return [Object]

suite[R]

Suite @return [Benchmark::IPS::NoopSuite]

time[RW]

Calculation time setter and getter (in seconds). @return [Integer]

timing[R]

Storing Iterations in time period. @return [Hash]

warmup[RW]

Warmup time setter and getter (in seconds). @return [Integer]

Public Class Methods

new(opts={}) click to toggle source

Instantiate the Benchmark::IPS::Job.

# File lib/benchmark/ips/job.rb, line 63
def initialize opts={}
  @list = []
  @run_single = false
  @json_path = false
  @compare = false
  @compare_order = :fastest
  @held_path = nil
  @held_results = nil

  @timing = Hash.new 1 # default to 1 in case warmup isn't run
  @full_report = Report.new

  # Default warmup and calculation time in seconds.
  @warmup = 2
  @time = 5
  @iterations = 1

  # Default statistical model
  @stats = :sd
  @confidence = 95

  self.quiet = false
end

Public Instance Methods

all_results_have_been_run?() click to toggle source
# File lib/benchmark/ips/job.rb, line 238
def all_results_have_been_run?
  @full_report.entries.size == @list.size
end
clear_held_results() click to toggle source
# File lib/benchmark/ips/job.rb, line 242
def clear_held_results
  File.delete @held_path if File.exist?(@held_path)
end
compare!(order: :fastest) click to toggle source

Run comparison utility.

# File lib/benchmark/ips/job.rb, line 121
def compare!(order: :fastest)
  @compare = true
  @compare_order = order
end
compare?() click to toggle source

Return true if job needs to be compared. @return [Boolean] Need to compare?

# File lib/benchmark/ips/job.rb, line 116
def compare?
  @compare
end
config(opts) click to toggle source

Job configuration options, set +@warmup+ and +@time+. @option opts [Integer] :warmup Warmup time. @option opts [Integer] :time Calculation time. @option iterations [Integer] :time Warmup and calculation iterations.

# File lib/benchmark/ips/job.rb, line 91
def config opts
  @warmup = opts[:warmup] if opts[:warmup]
  @time = opts[:time] if opts[:time]
  @suite = opts[:suite] if opts[:suite]
  @iterations = opts[:iterations] if opts[:iterations]
  @stats = opts[:stats] if opts[:stats]
  @confidence = opts[:confidence] if opts[:confidence]
  self.quiet = opts[:quiet] if opts.key?(:quiet)
  self.suite = opts[:suite]
end
create_report(label, measured_us, iter, samples, cycles) click to toggle source

Create report by add entry to +@full_report+. @param label [String] Report item label. @param measured_us [Integer] Measured time in microsecond. @param iter [Integer] Iterations. @param samples [Array<Float>] Sampled iterations per second. @param cycles [Integer] Number of Cycles. @return [Report::Entry] Entry with data.

# File lib/benchmark/ips/job.rb, line 393
def create_report(label, measured_us, iter, samples, cycles)
  @full_report.add_entry label, measured_us, iter, samples, cycles
end
create_stats(samples) click to toggle source
# File lib/benchmark/ips/job.rb, line 365
def create_stats(samples)
  case @stats
    when :sd
      Stats::SD.new(samples)
    when :bootstrap
      Stats::Bootstrap.new(samples, @confidence)
    else
      raise "unknown stats #{@stats}"
  end
end
cycles_per_100ms(time_msec, iters) click to toggle source

Calculate the cycles needed to run for approx 100ms, given the number of iterations to run the given time. @param [Float] time_msec Each iteration’s time in ms. @param [Integer] iters Iterations. @return [Integer] Cycles per 100ms.

# File lib/benchmark/ips/job.rb, line 190
def cycles_per_100ms time_msec, iters
  cycles = ((MICROSECONDS_PER_100MS / time_msec) * iters).to_i
  cycles <= 0 ? 1 : cycles
end
generate_json() click to toggle source

Generate json from +@full_report+.

# File lib/benchmark/ips/job.rb, line 382
def generate_json
  @full_report.generate_json @json_path if json?
end
hold!(held_path) click to toggle source

Hold after each iteration. @param held_path [String] File name to store hold file.

# File lib/benchmark/ips/job.rb, line 134
def hold!(held_path)
  @held_path = held_path
  @run_single = true
end
hold?() click to toggle source

Return true if results are held while multiple Ruby invocations @return [Boolean] Need to hold results between multiple Ruby invocations?

# File lib/benchmark/ips/job.rb, line 128
def hold?
  !!@held_path
end
item(label="", str=nil) { || ... } click to toggle source

Registers the given label and block pair in the job list. @param label [String] Label of benchmarked code. @param str [String] Code to be benchmarked. @param blk [Proc] Code to be benchmarked. @raise [ArgumentError] Raises if str and blk are both present. @raise [ArgumentError] Raises if str and blk are both absent.

# File lib/benchmark/ips/job.rb, line 172
def item(label="", str=nil, &blk) # :yield:
  if blk and str
    raise ArgumentError, "specify a block and a str, but not both"
  end

  action = str || blk
  raise ArgumentError, "no block or string" unless action

  @list.push Entry.new(label, action)
  self
end
Also aliased as: report
iterations_per_sec(cycles, time_us) click to toggle source

Calculate the interations per second given the number of cycles run and the time in microseconds that elapsed. @param [Integer] cycles Cycles. @param [Integer] time_us Time in microsecond. @return [Float] Iteration per second.

# File lib/benchmark/ips/job.rb, line 208
def iterations_per_sec cycles, time_us
  MICROSECONDS_PER_SECOND * (cycles.to_f / time_us.to_f)
end
json!(path="data.json") click to toggle source

Generate json to given path, defaults to “data.json”.

# File lib/benchmark/ips/job.rb, line 162
def json!(path="data.json")
  @json_path = path
end
json?() click to toggle source

Return true if job needs to generate json. @return [Boolean] Need to generate json?

# File lib/benchmark/ips/job.rb, line 157
def json?
  !!@json_path
end
load_held_results() click to toggle source
# File lib/benchmark/ips/job.rb, line 212
def load_held_results
  return unless @held_path && File.exist?(@held_path) && !File.zero?(@held_path)
  require "json"
  @held_results = {}
  JSON.load(IO.read(@held_path)).each do |result|
    @held_results[result['item']] = result
    create_report(result['item'], result['measured_us'], result['iter'],
                  create_stats(result['samples']), result['cycles'])
  end
end
quiet=(val) click to toggle source
# File lib/benchmark/ips/job.rb, line 102
def quiet=(val)
  @stdout = reporter(quiet: val)
end
report(label="", str=nil)
Alias for: item
reporter(quiet:) click to toggle source
# File lib/benchmark/ips/job.rb, line 110
def reporter(quiet:)
  quiet ? NoopReport.new : StdoutReport.new
end
run() click to toggle source
# File lib/benchmark/ips/job.rb, line 246
def run
  if @warmup && @warmup != 0 then
    @stdout.start_warming
    @iterations.times do
      run_warmup
    end
  end

  @stdout.start_running

  @iterations.times do |n|
    run_benchmark
  end

  @stdout.footer
end
run_benchmark() click to toggle source

Run calculation.

# File lib/benchmark/ips/job.rb, line 311
def run_benchmark
  @list.each do |item|
    next if run_single? && @held_results && @held_results.key?(item.label)

    @suite.running item.label, @time
    @stdout.running item.label, @time

    Timing.clean_env

    iter = 0

    measurements_us = []

    # Running this number of cycles should take around 100ms.
    cycles = @timing[item]

    target = Timing.add_second Timing.now, @time

    begin
      before = Timing.now
      item.call_times cycles
      after = Timing.now

      # If for some reason the timing said this took no time (O_o)
      # then ignore the iteration entirely and start another.
      iter_us = Timing.time_us before, after
      next if iter_us <= 0.0

      iter += cycles

      measurements_us << iter_us
    end while Timing.now < target

    final_time = before

    measured_us = measurements_us.inject(:+)

    samples = measurements_us.map { |time_us|
      iterations_per_sec cycles, time_us
    }

    rep = create_report(item.label, measured_us, iter, create_stats(samples), cycles)

    if (final_time - target).abs >= (@time.to_f * MAX_TIME_SKEW)
      rep.show_total_time!
    end

    @stdout.add_report rep, caller(1).first
    @suite.add_report rep, caller(1).first

    break if run_single?
  end
end
run_comparison() click to toggle source

Run comparison of entries in +@full_report+.

# File lib/benchmark/ips/job.rb, line 377
def run_comparison
  @full_report.run_comparison(@compare_order) if compare?
end
run_single?() click to toggle source

Return true if items are to be run one at a time. For the traditional hold, this is true @return [Boolean] Run just a single item?

# File lib/benchmark/ips/job.rb, line 151
def run_single?
  @run_single
end
run_warmup() click to toggle source

Run warmup.

# File lib/benchmark/ips/job.rb, line 264
def run_warmup
  @list.each do |item|
    next if run_single? && @held_results && @held_results.key?(item.label)

    @suite.warming item.label, @warmup
    @stdout.warming item.label, @warmup

    Timing.clean_env

    # Run for up to half of the configured warmup time with an increasing
    # number of cycles to reduce overhead and improve accuracy.
    # This also avoids running with a constant number of cycles, which a
    # JIT might speculate on and then have to recompile in #run_benchmark.
    before = Timing.now
    target = Timing.add_second before, @warmup / 2.0

    cycles = 1
    begin
      t0 = Timing.now
      item.call_times cycles
      t1 = Timing.now
      warmup_iter = cycles
      warmup_time_us = Timing.time_us(t0, t1)

      # If the number of cycles would go outside the 32-bit signed integers range
      # then exit the loop to avoid overflows and start the 100ms warmup runs
      break if cycles >= POW_2_30
      cycles *= 2
    end while Timing.now + warmup_time_us * 2 < target

    cycles = cycles_per_100ms warmup_time_us, warmup_iter
    @timing[item] = cycles

    # Run for the remaining of warmup in a similar way as #run_benchmark.
    target = Timing.add_second before, @warmup
    while Timing.now + MICROSECONDS_PER_100MS < target
      item.call_times cycles
    end

    @stdout.warmup_stats warmup_time_us, @timing[item]
    @suite.warmup_stats warmup_time_us, @timing[item]

    break if run_single?
  end
end
save!(held_path) click to toggle source

Save interim results. Similar to hold, but all reports are run The report label must change for each invocation. One way to achieve this is to include the version in the label. @param held_path [String] File name to store hold file.

# File lib/benchmark/ips/job.rb, line 143
def save!(held_path)
  @held_path = held_path
  @run_single = false
end
save_held_results() click to toggle source
# File lib/benchmark/ips/job.rb, line 223
def save_held_results
  return unless @held_path
  require "json"
  data = full_report.entries.map { |e|
    {
      'item' => e.label,
      'measured_us' => e.microseconds,
      'iter' => e.iterations,
      'samples' => e.samples,
      'cycles' => e.measurement_cycle
    }
  }
  IO.write(@held_path, JSON.generate(data) << "\n")
end
suite=(suite) click to toggle source
# File lib/benchmark/ips/job.rb, line 106
def suite=(suite)
  @suite = suite || Benchmark::IPS::NoopSuite.new
end
time_us(before, after) click to toggle source

Calculate the time difference of before and after in microseconds. @param [Time] before time. @param [Time] after time. @return [Float] Time difference of before and after.

# File lib/benchmark/ips/job.rb, line 199
def time_us before, after
  (after.to_f - before.to_f) * MICROSECONDS_PER_SECOND
end