class PushmiPullyu::CLI

CLI runner

Constants

COMMANDS

Public Class Methods

new() click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 17
def initialize
  PushmiPullyu.server_running = true # set to false by interrupt signal trap
  PushmiPullyu.reset_logger = false # set to true by SIGHUP trap
end

Public Instance Methods

parse(argv = ARGV) click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 22
def parse(argv = ARGV)
  opts = parse_options(argv)
  opts[:daemonize] = true if COMMANDS.include? argv[0]
  opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]

  PushmiPullyu.options = opts
end
run() click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 30
def run
  configure_rollbar
  begin
    if options[:daemonize]
      start_server_as_daemon
    else
      # If we're running in the foreground sync the output.
      $stdout.sync = $stderr.sync = true
      start_server
    end
  # rubocop:disable Lint/RescueException
  rescue Exception => e
    Rollbar.error(e)
    raise e
  end
  # rubocop:enable Lint/RescueException
end
start_server() click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 48
def start_server
  setup_signal_traps

  setup_log
  print_banner

  run_tick_loop
end

Private Instance Methods

configure_rollbar() click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 59
def configure_rollbar
  Rollbar.configure do |config|
    config.enabled = false unless options[:rollbar][:token].present?
    config.access_token = options[:rollbar][:token]
    config.use_exception_level_filters_default = true
    config.exception_level_filters['IOError'] = 'ignore'
    # add a filter after Rollbar has built the error payload but before it is delivered to the API,
    # in order to strip sensitive information out of certain error messages
    exception_message_transformer = proc do |payload|
      clean_message = payload[:exception][:message].sub(/http:\/\/.+:.+@(.+)\/aip\/v1\/(.*)/,
                                                        "http://\1/aip/v1/\2")
      payload[:exception][:message] = clean_message
      payload[:message] = clean_message
    end

    config.transform << exception_message_transformer

    if options[:rollbar][:proxy_host].present?
      config.proxy = {}
      config.proxy[:host] = options[:rollbar][:proxy_host]
      config.proxy[:port] = options[:rollbar][:proxy_port] if options[:rollbar][:proxy_port].present?
      config.proxy[:user] = options[:rollbar][:proxy_user] if options[:rollbar][:proxy_user].present?
      config.proxy[:password] = options[:rollbar][:proxy_password] if options[:rollbar][:proxy_password].present?
    end
  end
end
options() click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 86
def options
  PushmiPullyu.options
end
parse_config(config_file) click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 90
def parse_config(config_file)
  if File.exist?(config_file)
    YAML.safe_load(ERB.new(IO.read(config_file)).result).deep_symbolize_keys || {}
  else
    {}
  end
end
parse_options(argv) click to toggle source

Parse the options.

# File lib/pushmi_pullyu/cli.rb, line 99
def parse_options(argv)
  opts = {}

  @parsed_opts = OptionParser.new do |o|
    o.banner = 'Usage: pushmi_pullyu [options] [start|stop|restart|run]'
    o.separator ''
    o.separator 'Specific options:'

    o.on('-a', '--minimum-age AGE',
         Float, 'Minimum amount of time an item must spend in the queue, in seconds.') do |minimum_age|
      opts[:minimum_age] = minimum_age
    end

    o.on('-d', '--debug', 'Enable debug logging') do
      opts[:debug] = true
    end

    o.on('-r', '--rollbar-token TOKEN', 'Enable error reporting to Rollbar') do |token|
      if token.present?
        opts[:rollbar] = {}
        opts[:rollbar][:token] = token
      end
    end

    o.on '-C', '--config PATH', 'Path for YAML config file' do |config_file|
      opts[:config_file] = config_file
    end

    o.on('-L', '--logdir PATH', 'Path for directory to store log files') do |logdir|
      opts[:logdir] = logdir
    end

    o.on('-D', '--piddir PATH', 'Path for directory to store pid files') do |piddir|
      opts[:piddir] = piddir
    end

    o.on('-W', '--workdir PATH', 'Path for directory where AIP creation work takes place in') do |workdir|
      opts[:workdir] = workdir
    end

    o.on('-N', '--process_name NAME', 'Name of the application process') do |process_name|
      opts[:process_name] = process_name
    end

    o.on('-m', '--monitor', 'Start monitor process for a deamon') do
      opts[:monitor] = true
    end

    o.on('-q', '--queue NAME', 'Name of the queue to read from') do |queue|
      opts[:queue_name] = queue
    end

    o.separator ''
    o.separator 'Common options:'

    o.on_tail('-v', '--version', 'Show version') do
      puts "PushmiPullyu version: #{PushmiPullyu::VERSION}"
      exit
    end

    o.on_tail('-h', '--help', 'Show this message') do
      puts o
      exit
    end
  end.parse!(argv)

  ['config/pushmi_pullyu.yml', 'config/pushmi_pullyu.yml.erb'].each do |filename|
    opts[:config_file] ||= filename if File.exist?(filename)
  end

  opts
end
print_banner() click to toggle source
queue() click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 234
def queue
  @queue ||= PushmiPullyu::PreservationQueue.new(redis_url: options[:redis][:url],
                                                 queue_name: options[:queue_name],
                                                 age_at_least: options[:minimum_age])
end
rotate_logs() click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 178
def rotate_logs
  PushmiPullyu::Logging.reopen
  Daemonize.redirect_io(PushmiPullyu.application_log_file) if options[:daemonize]
  PushmiPullyu.reset_logger = false
end
run_preservation_cycle() click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 184
def run_preservation_cycle
  entity_json = JSON.parse(queue.wait_next_item)
  entity = {
    type: entity_json['type'],
    uuid: entity_json['uuid']
  }
  return unless entity[:type].present? && entity[:uuid].present?

  # add additional information about the error context to errors that occur while processing this item.
  Rollbar.scoped(entity_uuid: entity[:uuid]) do
    # Download AIP from Jupiter, bag and tar AIP directory and cleanup after
    # block code
    PushmiPullyu::AIP.create(entity) do |aip_filename, aip_directory|
      # Push tarred AIP to swift API
      deposited_file = swift.deposit_file(aip_filename, options[:swift][:container])
      # Log successful preservation event to the log files
      PushmiPullyu::Logging.log_preservation_event(deposited_file, aip_directory)
    end
    # rubocop:disable Lint/RescueException
  rescue Exception => e
    Rollbar.error(e)
    logger.error(e)
    # TODO: we could re-raise here and let the daemon die on any preservation error, or just log the issue and
    # move on to the next item.
    # rubocop:enable Lint/RescueException
  end
end
run_tick_loop() click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 212
def run_tick_loop
  while PushmiPullyu.server_running?
    run_preservation_cycle
    rotate_logs if PushmiPullyu.reset_logger?
  end
end
setup_log() click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 219
def setup_log
  if options[:daemonize]
    PushmiPullyu::Logging.initialize_logger(PushmiPullyu.application_log_file)
  else
    logger.formatter = PushmiPullyu::Logging::SimpleFormatter.new
  end
  logger.level = ::Logger::DEBUG if options[:debug]
end
setup_signal_traps() click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 228
def setup_signal_traps
  Signal.trap('INT') { shutdown }
  Signal.trap('TERM') { shutdown }
  Signal.trap('HUP') { PushmiPullyu.reset_logger = true }
end
shutdown() click to toggle source

On first call of shutdown, this will gracefully close the main run loop which let's the program exit itself. Calling shutdown again will force shutdown the program

# File lib/pushmi_pullyu/cli.rb, line 251
def shutdown
  if !PushmiPullyu.server_running?
    exit!(1)
  else
    # using stderr instead of logger as it uses an underlying mutex which is not allowed inside trap contexts.
    warn 'Exiting...  Interrupt again to force quit.'
    PushmiPullyu.server_running = false
  end
end
start_server_as_daemon() click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 261
def start_server_as_daemon
  require 'daemons'

  pwd = Dir.pwd # Current directory is changed during daemonization, so store it

  opts = {
    ARGV: @parsed_opts,
    dir: options[:piddir],
    dir_mode: :normal,
    monitor: options[:monitor],
    log_output: true,
    log_dir: File.expand_path(options[:logdir]),
    logfilename: File.basename(PushmiPullyu.application_log_file),
    output_logfilename: File.basename(PushmiPullyu.application_log_file)
  }

  Daemons.run_proc(options[:process_name], opts) do |*_argv|
    Dir.chdir(pwd)
    start_server
  end
end
swift() click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 240
def swift
  @swift ||= PushmiPullyu::SwiftDepositer.new(username: options[:swift][:username],
                                              password: options[:swift][:password],
                                              tenant: options[:swift][:tenant],
                                              project_name: options[:swift][:project_name],
                                              project_domain_name: options[:swift][:project_domain_name],
                                              auth_url: options[:swift][:auth_url])
end