module BatchExperiment

# File lib/batch_experiment.rb, line 201
def self.batch(commands, conf)
  # Throw exceptions if required configurations aren't provided.
  if !conf[:cpus_available] then
    fail ArgumentError, 'conf[:cpus_available] not set'
  end
  fail ArgumentError, 'conf[:timeout] not set' unless conf[:timeout]

  # Initialize optional configurations with default values if they weren't
  # provided. Don't change the conf argument, only our version of conf.
  conf = conf.clone
  conf[:time_fmt]         ||= 'ext_time: %e\\next_mem: %M\\n'
  conf[:out_ext]          ||= '.out'
  conf[:err_ext]          ||= '.err'
  conf[:run_ext]          ||= '.run'
  conf[:busy_loop_sleep]  ||= 0.1
  conf[:post_timeout]     ||= 5
  conf[:converter]        ||= BatchExperiment::Comm2FnameConverter.new
  conf[:skip_done_comms]    = true if conf[:skip_done_comms].nil?
  conf[:cwd]              ||= './'
  conf[:output_dir]       ||= './'

  # Initialize main variables
  free_cpus = conf[:cpus_available].clone
  comms_running = []
  cpu = nil
  comms_executed = []

  commands.each do | command |
    commfname = conf[:converter].call(command)
    out_fname = conf[:output_dir] + commfname + conf[:out_ext]
    err_fname = conf[:output_dir] + commfname + conf[:err_ext]
    run_fname = conf[:output_dir] + commfname + conf[:run_ext]

    if conf[:skip_done_comms] && File.exists?(run_fname)
      puts "Found file: #{commfname} -- skipping command: #{command}"
      STDOUT.flush
      next
    else
      if File.exists? out_fname then File.delete out_fname end
      if File.exists? err_fname then File.delete err_fname end
      if File.exists? run_fname then File.delete run_fname end
    end

    puts "Next command in the queue: #{command}"
    STDOUT.flush

    while free_cpus.empty? do
      sleep conf[:busy_loop_sleep]
      update_finished(free_cpus, comms_running, comms_executed)
    end

    cpu = free_cpus.pop

    cproc = ChildProcess.build(
      'taskset', '-c', cpu.to_s,
      'time', '-f', conf[:time_fmt], '--append', '-o', run_fname,
      'timeout', '--preserve-status', '-k', "#{conf[:post_timeout]}s",
        "#{conf[:timeout]}s",
      'sh', '-c', command
    )

    cproc.cwd = conf[:cwd]

    out = File.open(out_fname, 'w')
    err = File.open(err_fname, 'w')
    cproc.io.stdout = out
    cproc.io.stderr = err

    date_before = Time.now
    cproc.start

    comms_running << {
      proc: cproc,
      cpu: cpu,
      command: command,
      date_before: date_before,
      out_file: out,
      err_file: err,
      run_fname: run_fname,
    }

    puts "The command was assigned to cpu#{cpu}."
    STDOUT.flush
  end

  until comms_running.empty? do
    sleep conf[:busy_loop_sleep]
    update_finished(free_cpus, comms_running, comms_executed)
  end

  comms_executed
end

# File lib/batch_experiment.rb, line 429
def self.experiment(comms_info, batch_conf, conf, files)
  # Throw exceptions if required configurations aren't provided.
  fail 'conf[:csvfname] is not defined' unless conf[:csvfname]

  # Initialize optional configurations with default values if they weren't
  # provided. Don't change the conf argument, only our version of conf.
  conf = conf.clone
  conf[:separator]    ||= ';'
  conf[:qt_runs]      ||= 1
  conf[:comms_order]  ||= :by_comm
  conf[:rng]          ||= Random.new(42)
  #conf[:skip_commands] defaults to false/nil

  # Get some of the batch config that we use inside here too.
  run_ext         = batch_conf[:run_ext]        || '.run'
  out_ext         = batch_conf[:out_ext]        || '.out'
  output_dir      = batch_conf[:output_dir]     || './'
  converter = batch_conf[:converter].clone unless batch_conf[:converter].nil?
  converter ||= BatchExperiment::Comm2FnameConverter.new

  # Expand all commands, combining command templates and files.
  comms_sets = []
  comms_info.each do | comm_info |
    comms_sets << gencommff(comm_info, files)
  end

  expanded_comms = comms_sets.map { | h | h.keys }
  # If each command should be run more than once...
  if conf[:qt_runs] > 1
    # ... we replace each single command by an array of qt_runs copies,
    # and then flatten the parent array.
    expanded_comms.map! do | a |
      a.map! { | c | Array.new(conf[:qt_runs], c) }.flatten!
    end
  end

  # At this moment the expanded_comms is an array of arrays, each internal
  # array has all the expanded commands of the one single command template
  # over all the files.
  # After the code block below, the expanded_comms will be an one-level array
  # of the expanded commands, in the order they will be executed.
  expanded_comms = case conf[:comms_order]
  when :by_comm # all runs of the first command template first
    expanded_comms.flatten!
  when :by_file # all runs over the first file first
    intercalate(expanded_comms)
  when :random  # a random order
    expanded_comms.flatten!.shuffle!(random: conf[:rng])
  end

  # Execute the commands (or not).
  ret = batch(expanded_comms, batch_conf) unless conf[:skip_commands]

  # Build header (first csv line, column names).
  header = ['algorithm', 'filename', 'run_number']
  header << merge_headers(comms_info.map { | c | c[:extractor].names })
  header = header.join(conf[:separator])

  # We need to merge the union of all comms_sets to query it.
  comm2origin = {}
  comms_sets.each do | h |
    comm2origin.merge!(h) do | k, v, v2 |
      puts "WARNING: The command expansion '#{k}' was generated more than once. The first time was by the template '#{v[:comm]}' and the file '#{v[:file]}', and this time by template '#{v2[:comm]}' and the file '#{v2[:file]}'. Will report on CSV as this command was generated by the template '#{v[:comm]}' and the file '#{v[:file]}'."
      v
    end
  end

  # Build body (inspect all output files and make csv lines).
  #
  # Body format: algorithm;filename;run_number;first extracted column; ...
  #
  # This means that the extractors have to agree on what is each column, two
  # different extractors have to extract the same kind of data at each column
  # (the first field returned by all extractors has to be, for example, cpu
  # time, the same applies for the remaining fields).
  # If one extractor extract more fields than the others this is not a
  # problem, if the second biggest extractor (in number of fields extract)
  # will extract, for example, 4 fields, and the biggest extract 6 fields,
  # the first 4 fields extracted by the biggest extractor have to be the same
  # as the ones on the second-biggest extractor. This way, all the lines will
  # have the kind of data on the first four columns (not counting the
  # algorithm, filename and run_number ones), and only lines provenient from
  # the biggest extractor will have data on the fifth and sixth columns.
  body = [header]
  times_found = {}
  expanded_comms.each do | exp_comm |
    run_info   = comm2origin[exp_comm]
    algorithm  = run_info[:comm_info][:prefix]
    filename   = run_info[:filename]

    times_found[exp_comm] ||= 0
    times_found[exp_comm]  += 1
    run_number = times_found[exp_comm]

    curr_line = [algorithm, filename, run_number]

    partial_fname = converter.call(exp_comm)
    run_fname = output_dir + partial_fname + run_ext
    out_fname = output_dir + partial_fname + out_ext
    extractor = run_info[:comm_info][:extractor]

    if File.exists?(run_fname)
      run_info = File.open(run_fname, 'r') { | f | f.read }
      output = File.open(out_fname, 'r') { | f | f.read }
      # TODO: in the future change the extractors to receive
      # three inputs (out/err/run). If the runs create arbitrary files
      # with relevant info, the extractor will need to find, and open
      # them itself (i.e. it's not our job).
      curr_line << extractor.extract(output + "\n" + run_info)
    end

    body << curr_line.join(conf[:separator])
  end
  body = body.join(conf[:separator] + "\n")

  # Write CSV data into a CSV file.
  File.open(conf[:csvfname], 'w') { | f | f.write(body) }

  return ret
end

# File lib/batch_experiment.rb, line 306
def self.gencommff(comm_info, files)  #:nodoc
  ret = {}
  comm = comm_info[:command]
  patt = comm_info[:pattern]
  files.each do | f |
    ret[comm.gsub(patt, f)] = { comm_info: comm_info, filename: f }
  end
  ret
end

# File lib/batch_experiment.rb, line 324
def self.intercalate(xss)  #:nodoc
  ret = []
  xss = xss.map { | xs | xs.reverse }
  until xss.empty? do
    xss.delete_if do | xs |
      unless xs.empty?
        ret << xs.pop
      end
      xs.empty?
    end
  end
  ret
end

# File lib/batch_experiment.rb, line 347
def self.merge_headers(headers)  #:nodoc
  mer_size = headers.map { | h | h.size }.max
  merged_h = Array.new(mer_size)
  mer_size.times do | i |
    headers.each do | h |
      next if h.size < i
      if merged_h[i].nil?
        merged_h[i] = h[i]
      elsif merged_h[i] != h[i]
        raise ColumnSpecError, "Error: When using BatchExperiment::experiment"
          + " all the extractors have to agree on the columns they share." 
          + " In the specific case: the column nº #{i} was labeled as"
          + " '#{merged_h[i]}' on one extractor, and '#{h[i]}' on another,"
          + " this can be only a difference on notation ('time' vs 'Time'),"
          + " or can mean that in the same column two different kinds of data"
          + " are being presented. The program will be aborted. Check that."
      end
    end
  end
  merged_h
end

# File lib/batch_experiment.rb, line 101
def self.update_finished(free_cpus, comms_running, comms_executed) #:nodoc
  comms_running.delete_if do | job |
    # Don't call '#exited?' twice, store its value in a variable. If you call
    # it twice it's possible to remove it from the list of running commands
    # without freeing a cpu, what will mark the cpu as busy forever.
    exited = job[:proc].exited?
    if exited
      free_cpus.push(job[:cpu])
      job[:out_file].close
      job[:err_file].close
      File.open(job[:run_fname], 'a') do | f |
        f.write(
          "command: #{job[:command]}\n" +
          "date_before: #{job[:date_before].utc.to_s}\n" +
          "date_after: #{Time.now.utc.to_s}\n" +
          "hostname: #{Socket.gethostname}\n"
        )
      end
      comms_executed << job[:command]
    end
    exited # bool returned to delete_if
  end
end