class BenchmarkDriver::Runner::RubyStdout

Use stdout of ruby command

Constants

CommandFailure
Job

JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches “*::Job”

StdoutToMetrics

Public Class Methods

new(config:, output:, contexts:) click to toggle source

@param [BenchmarkDriver::Config::RunnerConfig] config @param [BenchmarkDriver::Output] output @param [BenchmarkDriver::Context] contexts

# File lib/benchmark_driver/runner/ruby_stdout.rb, line 71
def initialize(config:, output:, contexts:)
  @config = config
  @output = output
  @contexts = contexts
end

Public Instance Methods

run(jobs) click to toggle source

This method is dynamically called by `BenchmarkDriver::JobRunner.run` @param [Array<BenchmarkDriver::Default::Job>] jobs

# File lib/benchmark_driver/runner/ruby_stdout.rb, line 79
def run(jobs)
  if @config.alternate
    alternated_run(jobs)
  else
    incremental_run(jobs)
  end
end

Private Instance Methods

alternated_run(jobs) click to toggle source

Special mode. Execution order: RubyA, RubyB, …, RubyA, RubyB, …

# File lib/benchmark_driver/runner/ruby_stdout.rb, line 90
def alternated_run(jobs)
  metric = jobs.first.metrics.first

  @output.with_benchmark do
    jobs.each do |job|
      @output.with_job(name: job.name) do
        # Running benchmarks in an alternated manner is NOT compatible with two things:
        #   * Output plugins. They expect RubyA, RubyA, RubyB, RubyB, ...
        #   * BenchmarkDriver::Repeater. It should be used for results of the same condition.
        #
        # Therefore, we run all benchmarks with executables alternated first here, and then
        # aggregate the results as if the same executable were repeated in a row.
        context_results = Hash.new do |hash, context|
          hash[context] = []
        end
        jobs.each do |job|
          @config.repeat_count.times do
            @contexts.each do |context|
              context_results[context] << run_job(job, exec: context.executable)
            end
          end
        end

        # Aggregate reslts by BenchmarkDriver::Repeater and pass them to output.
        @contexts.each do |context|
          repeat_params = { config: @config, larger_better: metric.larger_better }
          result = BenchmarkDriver::Repeater.with_repeat(**repeat_params) do
            context_results[context].shift
          end
          value, environment = result.value

          exec = context.executable
          @output.with_context(name: exec.name, executable: exec) do
            @output.report(
              values: { metric => value },
              all_values: { metric => result.all_values },
              environment: environment,
            )
          end
        end
      end
    end
  end
end
execute(*args) click to toggle source
# File lib/benchmark_driver/runner/ruby_stdout.rb, line 195
def execute(*args)
  stdout, stderr, status = Open3.capture3(*args)
  unless status.success?
    raise CommandFailure.new("Failed to execute: #{args.shelljoin} (status: #{$?.exitstatus}):\n\n[stdout]:\n#{stdout}\n[stderr]:\n#{stderr}")
  end
  stdout
end
incremental_run(jobs) click to toggle source

Default mode. Execution order: RubyA, RubyA, RubyB, RubyB, …

# File lib/benchmark_driver/runner/ruby_stdout.rb, line 136
def incremental_run(jobs)
  metric = jobs.first.metrics.first

  @output.with_benchmark do
    jobs.each do |job|
      @output.with_job(name: job.name) do
        @contexts.each do |context|
          exec = context.executable
          repeat_params = { config: @config, larger_better: metric.larger_better }
          result = BenchmarkDriver::Repeater.with_repeat(**repeat_params) do
            run_job(job, exec: exec)
          end
          value, environment = result.value

          @output.with_context(name: exec.name, executable: exec) do
            @output.report(
              values: { metric => value },
              all_values: { metric => result.all_values },
              environment: environment,
            )
          end
        end
      end
    end
  end
end
run_job(job, exec:) click to toggle source

Run a job and return what BenchmarkDriver::Repeater.with_repeat takes.

# File lib/benchmark_driver/runner/ruby_stdout.rb, line 164
def run_job(job, exec:)
  stdout = with_chdir(job.working_directory) do
    with_ruby_prefix(exec) { execute(*exec.command, *job.command) }
  end
  script = StdoutToMetrics.new(
    stdout: stdout,
    value_from_stdout: job.value_from_stdout,
    environment_from_stdout: job.environment_from_stdout,
  )
  [script.value, script.environment]
rescue CommandFailure => e
  $stderr.puts("\n```\n#{e.message}```\n")
  [BenchmarkDriver::Result::ERROR, {}]
end
with_chdir(working_directory, &block) click to toggle source
# File lib/benchmark_driver/runner/ruby_stdout.rb, line 187
def with_chdir(working_directory, &block)
  if working_directory
    Dir.chdir(working_directory) { block.call }
  else
    block.call
  end
end
with_ruby_prefix(executable, &block) click to toggle source
# File lib/benchmark_driver/runner/ruby_stdout.rb, line 179
def with_ruby_prefix(executable, &block)
  env = ENV.to_h.dup
  ENV['PATH'] = "#{File.dirname(executable.command.first)}:#{ENV['PATH']}"
  block.call
ensure
  ENV.replace(env)
end