class Upstreamstatus
Constants
- VERSION
Attributes
conf[R]
Public Class Methods
new()
click to toggle source
# File lib/upstreamstatus.rb, line 28 def initialize @conf = OpenStruct.new load_conf return unless opts[:notify] %w( sentry_dsn pagerduty_api_key pagerduty_rest_api_key pagerduty_api_url pagerduty_service_id ).each do |key| fail "Config missing #{key}" unless conf[key] end Unirest.default_header 'Authorization', "Token token=#{pagerduty_rest_api_key}" Unirest.default_header 'Content-type', 'application/json' Unirest.default_header 'Accept', 'application/vnd.pagerduty+json;version=2' Raven.configure do |config| config.dsn = sentry_dsn config.logger = logger end end
Public Instance Methods
current_status()
click to toggle source
# File lib/upstreamstatus.rb, line 91 def current_status @current_status ||= begin return fake_response if opts[:simulate] r = Unirest.get status_check_url unless (200..299).include?(r.code) fail "Error code: #{r.code}\n" \ "Headers: #{r.headers}" \ "Body: #{r.body}" end r.body end end
down_hosts()
click to toggle source
# File lib/upstreamstatus.rb, line 54 def down_hosts current_status['servers']['server'].select { |s| s['status'] != 'up' } end
run()
click to toggle source
# File lib/upstreamstatus.rb, line 58 def run clear_active_alerts exit 0 if down_hosts.empty? puts "Detected down hosts:\n" print_hosts down_hosts logger.info "Detected down hosts: #{down_hosts.to_json}" if opts[:notify] down_hosts.each do |host| notify( "Upstream host #{host['upstream']} listed as down", host ) end end exit 1 rescue Interrupt => e puts "Received #{e.class}" exit 99 rescue SignalException => e logger.info "Received: #{e.signm} (#{e.signo})" exit 2 rescue SystemExit => e exit e.status rescue Exception => e # Need to rescue "Exception" so that Sentry gets it Raven.capture_exception(e) if sentry_dsn logger.fatal e.message logger.fatal e.backtrace.join("\n") raise e end
Private Instance Methods
active_alert?(host)
click to toggle source
# File lib/upstreamstatus.rb, line 123 def active_alert?(host) active_alerts.find do |a| a['incident_key'] == "upstreamstatus #{Socket.gethostname} #{host['upstream']}" end end
active_alerts()
click to toggle source
# File lib/upstreamstatus.rb, line 129 def active_alerts @active_alerts ||= active_alerts_paged.select do |a| a['incident_key'] =~ /^upstreamstatus #{Socket.gethostname} .*/ end end
active_alerts_paged(offset = 0)
click to toggle source
# File lib/upstreamstatus.rb, line 135 def active_alerts_paged(offset = 0) r = Unirest.get( "#{pagerduty_api_url}/incidents?service_ids[]=#{pagerduty_service_id}&statuses[]=triggered&statuses[]=acknowledged&offset=#{offset}" ) fail "Result: #{r.inspect}" unless (200..299).include?(r.code) return [] if r.body['total'].nil? pointer = r.body['limit'] + offset r.body['incidents'] + (pointer < r.body['total'] ? active_alerts_paged(pointer) : []) end
clear_active_alerts()
click to toggle source
# File lib/upstreamstatus.rb, line 146 def clear_active_alerts return unless opts[:notify] active_alerts.reject { |a| down_hosts_incident_keys.include? a['incident_key'] }.each do |a| puts "Resolving incident: #{a['incident_key']}" pagerduty.get_incident(a['incident_key']).resolve end end
defaults()
click to toggle source
# File lib/upstreamstatus.rb, line 192 def defaults { 'status_check_url' => 'http://localhost:8069/status?format=json' } end
down_hosts_incident_keys()
click to toggle source
# File lib/upstreamstatus.rb, line 154 def down_hosts_incident_keys down_hosts.map { |host| "upstreamstatus #{Socket.gethostname} #{host['upstream']}" } end
fake_response()
click to toggle source
# File lib/upstreamstatus.rb, line 196 def fake_response { 'servers' => { 'total' => 2, 'generation' => 99, 'server' => [ { 'index' => 0, 'upstream' => 'testupstream0', 'name' => '10.0.0.1 =>8080', 'status' => 'up', 'rise' => 10_459, 'fall' => 0, 'type' => 'http', 'port' => 0 }, { 'index' => 1, 'upstream' => 'testupstream1', 'name' => '10.0.0.2 =>8080', 'status' => 'down', 'rise' => 10_029, 'fall' => 0, 'type' => 'http', 'port' => 0 }, { 'index' => 2, 'upstream' => 'testupstream2', 'name' => '10.0.0.2 =>8080', 'status' => 'down', 'rise' => 10_029, 'fall' => 0, 'type' => 'http', 'port' => 0 } ] } } end
load_conf()
click to toggle source
# File lib/upstreamstatus.rb, line 186 def load_conf conf_file = '/etc/upstreamstatus.yml' yaml_conf = File.exist?(conf_file) ? YAML.load_file(conf_file) : {} defaults.merge(yaml_conf) end
logger()
click to toggle source
# File lib/upstreamstatus.rb, line 158 def logger @logger ||= Logger.new(conf['log']).tap { |l| l.progname = 'upstreamstatus' } end
notify(msg, host)
click to toggle source
# File lib/upstreamstatus.rb, line 109 def notify(msg, host) if active_alert?(host['upstream']) puts "Already an active alert. Not sending anything (message: #{msg})" else puts "Notifying PagerDuty (message: #{msg})" pagerduty.trigger( msg, incident_key: "upstreamstatus #{Socket.gethostname} #{host['upstream']}", client: Socket.gethostname, details: host ) end end
opts()
click to toggle source
# File lib/upstreamstatus.rb, line 166 def opts @opts ||= Optimist.options do opt :notify, 'Notify alert service on failure', short: '-n', default: false opt :simulate, 'Simulate a failed server', short: '-s', default: false end end
pagerduty()
click to toggle source
# File lib/upstreamstatus.rb, line 162 def pagerduty @pagerduty ||= Pagerduty.new pagerduty_api_key end
print_hosts(hosts)
click to toggle source
# File lib/upstreamstatus.rb, line 179 def print_hosts(hosts) hosts.each do |host| host.each { |k, v| puts "#{k}: #{v}" } puts end end