class Benchmarker::Benchmark

Attributes

colorize[R]
extra[R]
filter[R]
inverse[R]
iter[R]
loop[R]
outfile[R]
quiet[R]
sleep[R]
title[R]
width[R]

Public Class Methods

new(title: nil, width: 30, loop: 1, iter: 1, extra: 0, inverse: false, outfile: nil, quiet: false, colorize: nil, sleep: nil, filter: nil) click to toggle source
# File lib/benchmarker.rb, line 38
def initialize(title: nil, width: 30, loop: 1, iter: 1, extra: 0, inverse: false, outfile: nil, quiet: false, colorize: nil, sleep: nil, filter: nil)
  @title    = title
  @width    = width   || 30
  @loop     = loop    || 1
  @iter     = iter    || 1
  @extra    = extra   || 0
  @inverse  = inverse || false
  @outfile  = outfile
  @quiet    = quiet   || false
  @colorize = colorize
  @sleep    = sleep
  @filter   = filter
  if filter
    #; [!0mz0f] error when filter string is invalid format.
    filter =~ /^(task|tag)(!?=+)(.*)/  or
      raise ArgumentError.new("#{filter}: invalid filter.")
    #; [!xo7bq] error when filter operator is invalid.
    $2 == '=' || $2 == '!='  or
      raise ArgumentError.new("#{filter}: expected operator is '=' or '!='.")
  end
  @entries  = []    # [[Task, Resutl]]
  @jdata    = {}
  @hooks    = {}    # {before: Proc, after: Proc, ...}
  @empty_task = nil
end

Public Instance Methods

clear() click to toggle source
# File lib/benchmarker.rb, line 66
def clear()
  #; [!phqdn] clears benchmark result and JSON data.
  @entries.each {|_, result| result.clear() }
  @jdata = {}
  self
end
define_hook(key, &block) click to toggle source
# File lib/benchmarker.rb, line 99
def define_hook(key, &block)
  #; [!2u53t] register proc object with symbol key.
  @hooks[key] = block
  self
end
run(warmup: false) click to toggle source
# File lib/benchmarker.rb, line 112
def run(warmup: false)
  #; [!0fo0l] runs benchmark tasks and reports result.
  report_environment()
  filter_tasks()
  #; [!2j4ks] calls 'before_all' hook.
  call_hook(:before_all)
  begin
    if warmup
      #; [!6h26u] runs preriminary round when `warmup: true` provided.
      _ignore_output { invoke_tasks() }
      clear()
    end
    invoke_tasks()
  #; [!w1rq7] calls 'after_all' hook even if error raised.
  ensure
    call_hook(:after_all)
  end
  ignore_skipped_tasks()
  report_minmax()
  report_average()
  report_stats()
  write_outfile()
  nil
end
scope(&block) click to toggle source
# File lib/benchmarker.rb, line 73
def scope(&block)
  #; [!wrjy0] creates wrapper object and yields block with it as self.
  #; [!6h24d] passes benchmark object as argument of block.
  scope = Scope.new(self)
  scope.instance_exec(self, &block)
  #; [!y0uwr] returns self.
  self
end

Private Instance Methods

__invoke(task, task_name, validator, quiet) click to toggle source
# File lib/benchmarker.rb, line 237
def __invoke(task, task_name, validator, quiet)
  print "%-#{@width}s " % task_name unless quiet
  $stdout.flush()                   unless quiet
  #; [!fv4cv] skips task invocation if skip reason is specified.
  return nil if task.skip?
  #; [!hbass] calls 'before' hook with task name and tag.
  call_hook(:before, task.name, task.tag)
  #; [!6g36c] invokes task with validator if validator defined.
  begin
    timeset = task.invoke(@loop, &validator)
    return timeset
  #; [!7960c] calls 'after' hook with task name and tag even if error raised.
  ensure
    call_hook(:after, task_name, task.tag)
  end
end
_calc_average() click to toggle source
# File lib/benchmarker.rb, line 321
def _calc_average()
  #; [!qu29s] calculates average of real times for each task.
  rows = @entries.collect {|task, result|
    avg_timeset = result.calc_average()
    [task.name] + avg_timeset.to_a.collect {|x| ("%9.4f" % x).to_f }
  }
  #; [!jxf28] sets average results into JSON data.
  @jdata[:Average] = rows
  return rows
end
_ignore_output() { || ... } click to toggle source
# File lib/benchmarker.rb, line 139
def _ignore_output(&b)
  #; [!wazs7] ignores output in block argument.
  require 'stringio'
  bkup, $stdout = $stdout, StringIO.new
  begin
    yield
  ensure
    $stdout = bkup
  end
end
_remove_minmax() click to toggle source
# File lib/benchmarker.rb, line 277
def _remove_minmax()
  #; [!uxe7e] removes best and worst results if 'extra' option specified.
  tuples = []
  @entries.each do |task, result|
    removed_list = result.remove_minmax(@extra)
    tuples << [task.name, removed_list]
  end
  #; [!is6ll] returns removed min and max data.
  rows = []
  tuples.each do |task_name, removed_list|
    removed_list.each_with_index do |(min_t, min_idx, max_t, max_idx), i|
      task_name = nil if i > 0
      min_t2 = ("%9.4f" % min_t).to_f
      max_t2 = ("%9.4f" % max_t).to_f
      rows << [task_name, min_t2, "(##{min_idx})", max_t2, "(##{max_idx})"]
    end
  end
  #; [!xwddz] sets removed best and worst results into JSON data.
  @jdata[:RemovedMinMax] = rows
  return rows
end
_render_average(rows) click to toggle source
# File lib/benchmarker.rb, line 332
def _render_average(rows)
  #; [!j9wlv] returns rendered string.
  buf = ["\n"]
  heading = "## Average of #{@iter}"
  heading += " (=#{@iter + 2 * @extra}-2*#{@extra})" if @extra > 0
  buf << "%-#{@width+4}s %5s %9s %9s %9s\n" % [heading, 'user', 'sys', 'total', 'real']
  rows.each do |row|
    #buf << "%-#{@width}s %9.4f %9.4f %9.4f %9.4f\n" % row
    real = colorize_real('%9.4f' % row.pop())
    buf << "%-#{@width}s %9.4f %9.4f %9.4f %s\n" % (row + [real])
  end
  return buf.join()
end
_render_matrix(pairs) click to toggle source
# File lib/benchmarker.rb, line 387
def _render_matrix(pairs)
  #; [!2lu55] calculates ranking data and sets it into JSON data.
  rows = []
  pairs.each_with_index do |(task_name, sec), i|
    base = pairs[i][1]
    row = ["[#{i+1}] #{task_name}", ("%9.4f" % sec).to_f]
    pairs.each {|_, r| row << "%.1f%%" % (100.0 * r / base) }
    rows << row
  end
  @jdata[:Matrix] = rows
  #; [!rwfxu] returns rendered string of matrix.
  buf = ["\n"]
  heading = "## Matrix"
  s = "%-#{@width}s %9s" % [heading, 'real']
  (1..pairs.length).each {|i| s += " %8s" % "[#{i}]" }
  buf << "#{s}\n"
  rows.each do |task_name, real, *percents|
    s = "%-#{@width}s %s" % [task_name, colorize_real('%9.4f' % real)]
    percents.each {|p| s += " %8s" % p }
    buf << "#{s}\n"
  end
  return buf.join()
end
_render_minmax(rows) click to toggle source
# File lib/benchmarker.rb, line 299
def _render_minmax(rows)
  #; [!p71ax] returns rendered string.
  buf = ["\n"]
  heading = "## Removed Min & Max"
  buf << "%-#{@width+4}s %5s %9s %9s %9s\n" % [heading, 'min', 'iter', 'max', 'iter']
  rows.each do |row|
    #buf << "%-#{@width}s %9.4f %9s %9.4f %9s\n" % row
    task_name, min_t, min_i, max_t, max_i = row
    arr = [task_name, colorize_real('%9.4f' % min_t), colorize_iter('%9s' % min_i),
                      colorize_real('%9.4f' % max_t), colorize_iter('%9s' % max_i)]
    buf << "%-#{@width}s %s %s %s %s\n" % arr
  end
  return buf.join()
end
_render_ranking(pairs) click to toggle source
# File lib/benchmarker.rb, line 358
def _render_ranking(pairs)
  #; [!2lu55] calculates ranking data and sets it into JSON data.
  rows = []
  base = nil
  pairs.each do |task_name, sec|
    base ||= sec
    percent = 100.0 * base / sec
    barchart = '*' * (percent / 5.0).round()   # max 20 chars (=100%)
    loop = @inverse == true ? (@loop || 1) : (@inverse || @loop || 1)
    rows << [task_name, ("%.4f" % sec).to_f, "%.1f%%" % percent,
             "%.2f times/sec" % (loop / sec), barchart]
  end
  @jdata[:Ranking] = rows
  #; [!55x8r] returns rendered string of ranking.
  buf = ["\n"]
  heading = "## Ranking"
  if @inverse
    buf << "%-#{@width}s %9s%30s\n" % [heading, 'real', 'times/sec']
  else
    buf << "%-#{@width}s %9s\n"     % [heading, 'real']
  end
  rows.each do |task_name, sec, percent, inverse, barchart|
    s = @inverse ? "%20s" % inverse.split()[0] : barchart
    #buf << "%-#{@width}s %9.4f (%6s) %s\n" % [task_name, sec, percent, s]
    buf << "%-#{@width}s %s (%6s) %s\n" % [task_name, colorize_real('%9.4f' % sec), percent, s]
  end
  return buf.join()
end
call_hook(key, *args) click to toggle source
# File lib/benchmarker.rb, line 105
def call_hook(key, *args)
  #; [!0to2s] calls hook with arguments.
  fn = @hooks[key]
  fn.call(*args) if fn
end
colorize?() click to toggle source
# File lib/benchmarker.rb, line 426
def colorize?
  #; [!cy10n] returns true if '-c' option specified.
  #; [!e0gcz] returns false if '-C' option specified.
  #; [!6v90d] returns result of `Color.colorize?` if neither '-c' nor '-C' specified.
  return @colorize.nil? ? Color.colorize?() : @colorize
end
colorize_iter(s) click to toggle source
# File lib/benchmarker.rb, line 437
def colorize_iter(s)
  colorize?() ? Color.iter(s) : s
end
colorize_real(s) click to toggle source
# File lib/benchmarker.rb, line 433
def colorize_real(s)
  colorize?() ? Color.real(s) : s
end
filter_tasks() click to toggle source
# File lib/benchmarker.rb, line 150
def filter_tasks()
  #; [!g207d] do nothing when filter string is not provided.
  if @filter
    #; [!f1n1v] filters tasks by task name when filer string is 'task=...'.
    #; [!m79cf] filters tasks by tag value when filer string is 'tag=...'.
    @filter =~ /^(task|tag)(=|!=)(.*)/  or raise "** internal error"
    key = $1; op = $2; pattern = $3
    @entries = @entries.select {|task, _|
      val = key == 'tag' ? task.tag : task.name
      if val
        bool = [val].flatten.any? {|v| File.fnmatch(pattern, v, File::FNM_EXTGLOB) }
      else
        bool = false
      end
      #; [!0in0q] supports negative filter by '!=' operator.
      op == '!=' ? !bool : bool
    }
  end
  nil
end
ignore_skipped_tasks() click to toggle source
# File lib/benchmarker.rb, line 254
def ignore_skipped_tasks()
  #; [!5gpo7] removes skipped tasks and leaves other tasks.
  @entries = @entries.reject {|_, result| result.skipped? }
  nil
end
invoke_tasks() click to toggle source
# File lib/benchmarker.rb, line 171
def invoke_tasks()
  @jdata[:Results] = []
  #; [!c8yak] invokes tasks once if 'iter' option not specified.
  #; [!unond] invokes tasks multiple times if 'iter' option specified.
  #; [!wzvdb] invokes tasks 16 times if 'iter' is 10 and 'extra' is 3.
  n = @iter + 2 * @extra
  (1..n).each do |i|
    @jdata[:Results] << (rows = [])
    #; [!5axhl] prints result even on quiet mode if no 'iter' nor 'extra'.
    quiet = @quiet && n != 1
    #; [!yg9i7] prints result unless quiet mode.
    #; [!94916] suppresses result if quiet mode.
    #heading = n == 1 ? "##" : "## (##{i})"
    if n == 1
      heading = "##"
      space = " " * (@width - heading.length)
    else
      heading = "## " + colorize_iter("(##{i})")
      space = " " * (@width - "## (##{i})".length)
    end
    puts "" unless quiet
    #puts "%-#{@width}s %9s %9s %9s %9s" % [heading, 'user', 'sys', 'total', 'real'] unless quiet
    puts "%s%s %9s %9s %9s %9s" % [heading, space, 'user', 'sys', 'total', 'real'] unless quiet
    #; [!3hgos] invokes empty task at first if defined.
    if @empty_task && !@empty_task.skip?
      empty_timeset = __invoke(@empty_task, "(Empty)", nil, quiet)
      t = empty_timeset
      s = "%9.4f %9.4f %9.4f %9.4f" % [t.user, t.sys, t.total, t.real]
      #s = "%9.4f %9.4f %9.4f %s" % [t.user, t.sys, t.total, colorize_real('%9.4f' % t.real)]
      puts s unless quiet
      #; [!knjls] records result of empty loop into JSON data.
      rows << ["(Empty)"] + empty_timeset.to_a.collect {|x| ('%9.4f' % x).to_f }
      Kernel.sleep @sleep if @sleep
    else
      empty_timeset = nil
    end
    #; [!xf84h] invokes all tasks.
    @entries.each do |task, result|
      timeset = __invoke(task, task.name, @hooks[:validate], quiet)
      #; [!dyofw] prints reason if 'skip:' option specified.
      if task.skip?
        reason = task.skip
        result.skipped = reason
        puts "   # Skipped (reason: #{reason})" unless quiet
        #; [!ygpx0] records reason of skip into JSON data.
        rows << [task.name, nil, nil, nil, nil, reason]
        next
      end
      #; [!513ok] subtract timeset of empty loop from timeset of each task.
      if empty_timeset
        timeset -= empty_timeset           unless task.has_code?
        timeset -= empty_timeset.div(N_REPEAT) if task.has_code?
      end
      t = timeset
      #s = "%9.4f %9.4f %9.4f %9.4f" % [t.user, t.sys, t.total, t.real]
      s = "%9.4f %9.4f %9.4f %s" % [t.user, t.sys, t.total, colorize_real('%9.4f' % t.real)]
      puts s unless quiet
      result.add(timeset)
      #; [!ejxif] records result of each task into JSON data.
      rows << [task.name] + timeset.to_a.collect {|x| ('%9.4f' % x).to_f }
      #; [!vbhvz] sleeps N seconds after each task if `sleep` option specified.
      Kernel.sleep @sleep if @sleep
    end
  end
  nil
end
report_average() click to toggle source
# File lib/benchmarker.rb, line 314
def report_average()
  if @iter > 1 || @extra > 0
    rows = _calc_average()
    puts _render_average(rows)
  end
end
report_environment() click to toggle source
# File lib/benchmarker.rb, line 260
def report_environment()
  #; [!rx7nn] prints ruby version, platform, several options, and so on.
  s = "loop=#{@loop.inspect}, iter=#{@iter.inspect}, extra=#{@extra.inspect}"
  s += ", inverse=#{@inverse}" if @inverse
  kvs = [["title", @title], ["options", s]] + Misc.environment_info()
  puts kvs.collect {|k, v| "## %-16s %s\n" % ["#{k}:", v] }.join()
  @jdata[:Environment] = Hash.new(kvs)
  nil
end
report_minmax() click to toggle source
# File lib/benchmarker.rb, line 270
def report_minmax()
  if @extra > 0
    rows = _remove_minmax()
    puts _render_minmax(rows)
  end
end
report_stats() click to toggle source
# File lib/benchmarker.rb, line 346
def report_stats()
  #; [!0jn7d] sorts results by real sec.
  pairs = @entries.collect {|task, result|
    #real = @iter > 1 || @extra > 0 ? result.calc_average().real : result[0].real
    real = result.calc_average().real
    [task.name, real]
  }
  pairs = pairs.sort_by {|_, real| real }
  print _render_ranking(pairs)
  print _render_matrix(pairs)
end
write_outfile() click to toggle source
# File lib/benchmarker.rb, line 411
def write_outfile()
  #; [!o8ah6] writes result data into JSON file if 'outfile' option specified.
  if @outfile
    filename = @outfile
    require 'json'
    jstr = JSON.pretty_generate(@jdata, indent: '  ', space: ' ')
    if filename == '-'
      $stdout.puts(jstr)
    else
      File.write(filename, jstr)
    end
    jstr
  end
end