class PushmiPullyu::CLI
CLI
runner
Constants
- COMMANDS
Public Class Methods
new()
click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 17 def initialize PushmiPullyu.server_running = true # set to false by interrupt signal trap PushmiPullyu.reset_logger = false # set to true by SIGHUP trap end
Public Instance Methods
parse(argv = ARGV)
click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 22 def parse(argv = ARGV) opts = parse_options(argv) opts[:daemonize] = true if COMMANDS.include? argv[0] opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file] PushmiPullyu.options = opts end
run()
click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 30 def run configure_rollbar begin if options[:daemonize] start_server_as_daemon else # If we're running in the foreground sync the output. $stdout.sync = $stderr.sync = true start_server end # rubocop:disable Lint/RescueException rescue Exception => e Rollbar.error(e) raise e end # rubocop:enable Lint/RescueException end
start_server()
click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 48 def start_server setup_signal_traps setup_log print_banner run_tick_loop end
Private Instance Methods
configure_rollbar()
click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 59 def configure_rollbar Rollbar.configure do |config| config.enabled = false unless options[:rollbar][:token].present? config.access_token = options[:rollbar][:token] config.use_exception_level_filters_default = true config.exception_level_filters['IOError'] = 'ignore' # add a filter after Rollbar has built the error payload but before it is delivered to the API, # in order to strip sensitive information out of certain error messages exception_message_transformer = proc do |payload| clean_message = payload[:exception][:message].sub(/http:\/\/.+:.+@(.+)\/aip\/v1\/(.*)/, "http://\1/aip/v1/\2") payload[:exception][:message] = clean_message payload[:message] = clean_message end config.transform << exception_message_transformer if options[:rollbar][:proxy_host].present? config.proxy = {} config.proxy[:host] = options[:rollbar][:proxy_host] config.proxy[:port] = options[:rollbar][:proxy_port] if options[:rollbar][:proxy_port].present? config.proxy[:user] = options[:rollbar][:proxy_user] if options[:rollbar][:proxy_user].present? config.proxy[:password] = options[:rollbar][:proxy_password] if options[:rollbar][:proxy_password].present? end end end
options()
click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 86 def options PushmiPullyu.options end
parse_config(config_file)
click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 90 def parse_config(config_file) if File.exist?(config_file) YAML.safe_load(ERB.new(IO.read(config_file)).result).deep_symbolize_keys || {} else {} end end
parse_options(argv)
click to toggle source
Parse the options.
# File lib/pushmi_pullyu/cli.rb, line 99 def parse_options(argv) opts = {} @parsed_opts = OptionParser.new do |o| o.banner = 'Usage: pushmi_pullyu [options] [start|stop|restart|run]' o.separator '' o.separator 'Specific options:' o.on('-a', '--minimum-age AGE', Float, 'Minimum amount of time an item must spend in the queue, in seconds.') do |minimum_age| opts[:minimum_age] = minimum_age end o.on('-d', '--debug', 'Enable debug logging') do opts[:debug] = true end o.on('-r', '--rollbar-token TOKEN', 'Enable error reporting to Rollbar') do |token| if token.present? opts[:rollbar] = {} opts[:rollbar][:token] = token end end o.on '-C', '--config PATH', 'Path for YAML config file' do |config_file| opts[:config_file] = config_file end o.on('-L', '--logdir PATH', 'Path for directory to store log files') do |logdir| opts[:logdir] = logdir end o.on('-D', '--piddir PATH', 'Path for directory to store pid files') do |piddir| opts[:piddir] = piddir end o.on('-W', '--workdir PATH', 'Path for directory where AIP creation work takes place in') do |workdir| opts[:workdir] = workdir end o.on('-N', '--process_name NAME', 'Name of the application process') do |process_name| opts[:process_name] = process_name end o.on('-m', '--monitor', 'Start monitor process for a deamon') do opts[:monitor] = true end o.on('-q', '--queue NAME', 'Name of the queue to read from') do |queue| opts[:queue_name] = queue end o.separator '' o.separator 'Common options:' o.on_tail('-v', '--version', 'Show version') do puts "PushmiPullyu version: #{PushmiPullyu::VERSION}" exit end o.on_tail('-h', '--help', 'Show this message') do puts o exit end end.parse!(argv) ['config/pushmi_pullyu.yml', 'config/pushmi_pullyu.yml.erb'].each do |filename| opts[:config_file] ||= filename if File.exist?(filename) end opts end
queue()
click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 234 def queue @queue ||= PushmiPullyu::PreservationQueue.new(redis_url: options[:redis][:url], queue_name: options[:queue_name], age_at_least: options[:minimum_age]) end
rotate_logs()
click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 178 def rotate_logs PushmiPullyu::Logging.reopen Daemonize.redirect_io(PushmiPullyu.application_log_file) if options[:daemonize] PushmiPullyu.reset_logger = false end
run_preservation_cycle()
click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 184 def run_preservation_cycle entity_json = JSON.parse(queue.wait_next_item) entity = { type: entity_json['type'], uuid: entity_json['uuid'] } return unless entity[:type].present? && entity[:uuid].present? # add additional information about the error context to errors that occur while processing this item. Rollbar.scoped(entity_uuid: entity[:uuid]) do # Download AIP from Jupiter, bag and tar AIP directory and cleanup after # block code PushmiPullyu::AIP.create(entity) do |aip_filename, aip_directory| # Push tarred AIP to swift API deposited_file = swift.deposit_file(aip_filename, options[:swift][:container]) # Log successful preservation event to the log files PushmiPullyu::Logging.log_preservation_event(deposited_file, aip_directory) end # rubocop:disable Lint/RescueException rescue Exception => e Rollbar.error(e) logger.error(e) # TODO: we could re-raise here and let the daemon die on any preservation error, or just log the issue and # move on to the next item. # rubocop:enable Lint/RescueException end end
run_tick_loop()
click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 212 def run_tick_loop while PushmiPullyu.server_running? run_preservation_cycle rotate_logs if PushmiPullyu.reset_logger? end end
setup_log()
click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 219 def setup_log if options[:daemonize] PushmiPullyu::Logging.initialize_logger(PushmiPullyu.application_log_file) else logger.formatter = PushmiPullyu::Logging::SimpleFormatter.new end logger.level = ::Logger::DEBUG if options[:debug] end
setup_signal_traps()
click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 228 def setup_signal_traps Signal.trap('INT') { shutdown } Signal.trap('TERM') { shutdown } Signal.trap('HUP') { PushmiPullyu.reset_logger = true } end
shutdown()
click to toggle source
On first call of shutdown, this will gracefully close the main run loop which let's the program exit itself. Calling shutdown again will force shutdown the program
# File lib/pushmi_pullyu/cli.rb, line 251 def shutdown if !PushmiPullyu.server_running? exit!(1) else # using stderr instead of logger as it uses an underlying mutex which is not allowed inside trap contexts. warn 'Exiting... Interrupt again to force quit.' PushmiPullyu.server_running = false end end
start_server_as_daemon()
click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 261 def start_server_as_daemon require 'daemons' pwd = Dir.pwd # Current directory is changed during daemonization, so store it opts = { ARGV: @parsed_opts, dir: options[:piddir], dir_mode: :normal, monitor: options[:monitor], log_output: true, log_dir: File.expand_path(options[:logdir]), logfilename: File.basename(PushmiPullyu.application_log_file), output_logfilename: File.basename(PushmiPullyu.application_log_file) } Daemons.run_proc(options[:process_name], opts) do |*_argv| Dir.chdir(pwd) start_server end end
swift()
click to toggle source
# File lib/pushmi_pullyu/cli.rb, line 240 def swift @swift ||= PushmiPullyu::SwiftDepositer.new(username: options[:swift][:username], password: options[:swift][:password], tenant: options[:swift][:tenant], project_name: options[:swift][:project_name], project_domain_name: options[:swift][:project_domain_name], auth_url: options[:swift][:auth_url]) end