class BioDSL::Pipeline

Pipeline class

Attributes

commands[RW]
complete[RW]

Public Class Methods

new() click to toggle source

Pipeline class constructor.

# File lib/BioDSL/pipeline.rb, line 54
def initialize
  @commands = []      # Array of Commands in the Pipeline.
  @options  = {}      # Options hash.
  @enums    = [[]]    # Array of Enumerators.
  @complete = false   # Flag denoting if run was completed.
end

Public Instance Methods

+(other) click to toggle source

Method that adds two Pipelines and return a new Pipeline.

# File lib/BioDSL/pipeline.rb, line 79
def +(other)
  unless other.is_a?(BioDSL::Pipeline)
    fail PipelineError, "Not a pipeline: #{other.inspect}"
  end

  p = self.class.new
  p << self
  p << other
end
<<(other) click to toggle source

Method for merging one pipeline onto another.

@param other [Pipeline] Pipeline to merge.

@return [self].

# File lib/BioDSL/pipeline.rb, line 71
def <<(other)
  other.commands.map { |command| commands << command }
  other.status.map   { |status|  self.status << status }

  self
end
pop() click to toggle source

Removes last command from a Pipeline and returns a new Pipeline with this command.

# File lib/BioDSL/pipeline.rb, line 91
def pop
  p = BioDSL::Pipeline.new
  p.commands = [@commands.pop]
  p
end
run(options = {}) click to toggle source

Run all the commands in the Pipeline.

@param options [Hash] @option options [Boolean] :verbose (false) Enable verbose output.

@raise [PipelineError] If no commands are added to the pipeline.

@return [self]

# File lib/BioDSL/pipeline.rb, line 105
def run(options = {})
  prime_variables(options)

  fail BioDSL::PipelineError, 'Empty pipeline' if @commands.empty?

  @options = options

  check_options
  command_runner
  print_status
  send_email(self)
  save_report
  log_ok

  self
rescue => exception
  exit_gracefully(exception)
ensure
  save_history
end
size() click to toggle source

@return [Integer] The size or number of commands in a pipeline.

# File lib/BioDSL/pipeline.rb, line 62
def size
  @commands.size
end
status() click to toggle source

Return a list of all status hashes from the commands.

@return [Array] List of status hashes.

# File lib/BioDSL/pipeline.rb, line 129
def status
  @commands.each_with_object([]) do |e, a|
    if @complete
      e.calc_time_elapsed
      e.calc_delta
    end

    a << e.status
  end
end
to_s() click to toggle source

Format a Pipeline to a pretty string which is returned.

# File lib/BioDSL/pipeline.rb, line 141
def to_s
  command_strings = %w(BD new)

  @commands.each { |command| command_strings << command.to_s }

  if @complete
    if @options.empty?
      command_strings << 'run'
    else
      command_strings << "run(#{options_string})"
    end
  end

  command_strings.join('.')
end

Private Instance Methods

check_options() click to toggle source

Check all run options.

# File lib/BioDSL/pipeline.rb, line 225
def check_options
  options_allowed(@options, :debug, :verbose, :email, :progress, :subject,
                  :input, :output, :output_dir, :report, :force)
  options_allowed_values(@options, debug: [true, false, nil])
  options_allowed_values(@options, verbose: [true, false, nil])
  options_conflict(@options, progress: :verbose)
  options_tie(@options, subject: :email)
  options_files_exist_force(@options, :report)
end
command_runner() click to toggle source

Run all commands.

# File lib/BioDSL/pipeline.rb, line 302
def command_runner
  return if @complete

  if @options[:progress]
    status_progress(@commands) { run_commands }
  else
    run_commands
  end

  @complete = true
end
exit_gracefully(exception) click to toggle source

Output exception message and possibly stack tracre to STDERR, log error message and exit with non-zero status.

# File lib/BioDSL/pipeline.rb, line 327
def exit_gracefully(exception)
  fail exception if BioDSL.test

  STDERR.puts "Error in run: #{exception.message}"
  STDERR.puts exception.backtrace if BioDSL.verbose
  log_error(exception)
  exit 2
end
method_missing(method, *args, &block) click to toggle source

Add a command to the pipeline. This is done by first requiring the relevant Class/Module and then calling the relevant command.

@param method [Symbol] Method name. @param args [Array] Method arguments. @param block [Proc] Method block.

@example Here we add the command ‘dump` to the pipeline.

Pipeline.new.dump
  # => self

@return [self]

Calls superclass method
# File lib/BioDSL/pipeline.rb, line 171
def method_missing(method, *args, &block)
  require_file(method)

  const = method.to_s.split('_').map(&:capitalize).join('')

  if BioDSL.const_defined? const
    options = args.first || {}
    options_load_rc(options, method)

    klass = BioDSL.const_get(const)
    klass.send(:include, OptionsHelper)
    klass.send(:include, StatusHelper)
    lmb = klass.send(:new, options).lmb

    @commands << Command.new(method, lmb, options)
  else
    super
  end

  self
end
options_string() click to toggle source

Generate a comma separated string from the options ensuring that values are in “” if need be.

Return [Array] List of options.

# File lib/BioDSL/pipeline.rb, line 340
def options_string
  options = []

  @options.each_pair do |key, value|
    if value.is_a? String
      options << %(#{key}: "#{value}")
    else
      options << %(#{key}: #{value})
    end
  end

  options.join(', ')
end
prefix_output_dir() click to toggle source

Create an output directory and prefix all output files in the commands with this directory.

# File lib/BioDSL/pipeline.rb, line 272
def prefix_output_dir
  return unless @options[:output_dir]

  unless File.exist?(@options[:output_dir])
    FileUtils.mkdir_p(@options[:output_dir])
  end

  @commands.each do |command|
    if (value = command.options[:output])
      command.options[:output] = File.join(@options[:output_dir], value)
    end
  end
end
prime_variables(options) click to toggle source

Set some global variables.

@param options [Hash] Options hash. @option options [Booleon] :debug Debug flag. @option options [Booleon] :verbose Verbose flag.

# File lib/BioDSL/pipeline.rb, line 319
def prime_variables(options)
  BioDSL.test    = ENV['BD_TEST']
  BioDSL.debug   = options[:debug]
  BioDSL.verbose = options[:verbose]
end
print_status() click to toggle source

Print status.

require_file(method) click to toggle source

Require a file form the lib/commands directory given a method name that must match the file name. E.g. ‘require_file(:dump)` requires the file `lib/commands/dump.rb`.

@param method [Symbol]

The name of the method.

@raise [Errno::ENOENT] If no such file was found.

# File lib/BioDSL/pipeline.rb, line 201
def require_file(method)
  return if BioDSL.const_defined? method.to_s.capitalize

  # FIXME
  # file = File.join('lib', 'BioDSL', 'commands', "#{method}.rb")
  # fail Errno::ENOENT, "No such file: #{file}" unless File.exist? file

  require File.join('BioDSL', 'commands', method.to_s)
end
run_add_enumerators() click to toggle source

Add enumerators to instance array.

# File lib/BioDSL/pipeline.rb, line 253
def run_add_enumerators
  @commands.each do |command|
    input  = @options[:input] || @enums.last
    @enums << Enumerator.new { |output| command.call(input, output) }
  end
end
run_commands() click to toggle source

Run all commands in the Pipeline.

# File lib/BioDSL/pipeline.rb, line 236
def run_commands
  prefix_output_dir
  run_time_start
  run_add_enumerators
  run_enumerate
end
run_enumerate() click to toggle source

Iterate through all enumerators.

# File lib/BioDSL/pipeline.rb, line 261
def run_enumerate
  if @options[:output]
    @enums.last.each { |record| @options[:output].write record }
    @options[:output].close   # TODO: this close is ugly here
  else
    @enums.last.each {}
  end
end
run_time_start() click to toggle source

Add start time to the status of all commands.

# File lib/BioDSL/pipeline.rb, line 244
def run_time_start
  time = Time.now

  @commands.each do |command|
    command.status[:time_start] = time
  end
end
save_report() click to toggle source

Save a HTML status report to file.

# File lib/BioDSL/pipeline.rb, line 287
def save_report
  return unless @options[:report]

  file = if @options[:output_dir]
           File.join(@options[:output_dir], @options[:report])
         else
           @options[:report]
         end

  File.open(file, 'w') do |ios|
    ios.puts BioDSL::HtmlReport.new(self).to_html
  end
end