class SysWatchdog

Constants

CRONJOB_PATH
DEFAULT_CONF_FILE
DEFAULT_LOG_FILE
INSTALL_TYPE_PATH
WORKING_DIR

Public Class Methods

new(conf_file: nil, log_file: nil) click to toggle source
# File lib/sys_watchdog/main.rb, line 8
def initialize conf_file: nil, log_file: nil
    log_file  ||= DEFAULT_LOG_FILE
    conf_file ||= DEFAULT_CONF_FILE

    @logger = WdLogger.new log_file
    parse_conf conf_file

    setup
end

Public Instance Methods

run(once: false) click to toggle source
# File lib/sys_watchdog/main.rb, line 18
def run once: false
    loop do
        @tests.each{|test| run_test test}
        return if once
        sleep 60
    end
end

Private Instance Methods

append_sys_info() click to toggle source
# File lib/sys_watchdog/notify.rb, line 44
def append_sys_info
    ret = "\n\n--------------- sys info ---------------"
    %Q(
        ps aux
        df -h
        uptime

    ).split("\n").map(&:strip).compact.reject(&:empty?).each do |cmd|
        cmd_result = `#{ cmd }`
        ret += "\n\n#{ cmd }:\n#{ cmd_result }"
    end
    ret
end
check_conf_file(conf_file) click to toggle source
# File lib/sys_watchdog/main.rb, line 56
def check_conf_file conf_file
    unless File.readable? conf_file
        raise "Conf file #{conf_file} not found or unreadable. Aborting." 
    end

    conf_stat = File.stat conf_file

    unless conf_stat.mode.to_s(8) =~ /0600$/
        raise "Conf file #{conf_file} must have mode 0600. Aborting." 
    end

    unless match_root_or_current_user(conf_stat)
        raise "Conf file #{conf_file} must have uid/gid set to root or to current running uid/gid. Aborting." 
    end
end
fail(test, output) click to toggle source
# File lib/sys_watchdog/main.rb, line 101
def fail test, output
    body = ""
    body += "output: #{output}" if output and not output.empty?
    notify "#{test.name} fail", body
end
notify(sub, body = "") click to toggle source
# File lib/sys_watchdog/notify.rb, line 4
def notify sub, body = ""
    begin
        send_mail sub, body if @conf.smtp_server
    rescue => e
        @logger.error e.desc
    end
    begin
        send_slack_msg "#{sub}\n#{body}" if @conf.slack_token
    rescue => e
        @logger.error e.desc
    end
end
notify_output_change(test, output) click to toggle source
# File lib/sys_watchdog/main.rb, line 94
def notify_output_change test, output
    if test.notify_on_output_change and test.previous_output != output
        notify "#{test.name} changed", "old: #{test.previous_output}\nnew: #{output}"
        test.previous_output = output
    end
end
parse_conf(conf_file) click to toggle source
# File lib/sys_watchdog/main.rb, line 43
def parse_conf conf_file
    check_conf_file conf_file

    conf = YAML.load_file conf_file
    conf.deep_symbolize_keys!

    @conf = OpenStruct.new conf[:config]

    @tests = conf[:tests].keys.map { |name|
        WdTest.new(name, conf[:tests][name], @logger)
    }
end
run_test(test, after_restore: false) click to toggle source
# File lib/sys_watchdog/main.rb, line 72
def run_test test, after_restore: false
    new_status, _exitstatus, output = test.run

    notify_output_change test, output

    return if new_status == test.status
    test.status = new_status
    
    if new_status
        notify "#{test.name} ok"
    else
        if test.restore_cmd and not after_restore
            test.restore
            run_test test, after_restore: true
        else
            fail test, output
        end
    end
rescue => e
    @logger.error e.desc
end
send_mail(sub, body) click to toggle source
# File lib/sys_watchdog/notify.rb, line 22
def send_mail sub, body
    @logger.info "Sending email: #{ sub }"

    body ||= ""
    body += append_sys_info
    
    conf_ = @conf

    mail = Mail.new do
      from     conf_.mail_from
      to       conf_.mail_to
      subject  "Watchdog #{conf_.name} [#{server_name}]: #{ sub }"
      body     body
    end

    mail.deliver!
end
send_slack_msg(msg) click to toggle source
# File lib/sys_watchdog/notify.rb, line 17
def send_slack_msg msg
    slack_client = Slack::Web::Client.new
    slack_client.chat_postMessage(channel: @conf[:slack_channel], text: "[#{server_name}] #{msg}", as_user: true)
end
server_name() click to toggle source
# File lib/sys_watchdog/notify.rb, line 40
def server_name
    @conf.server_name || `hostname`.strip
end
setup() click to toggle source
# File lib/sys_watchdog/main.rb, line 28
def setup
    if @conf.slack_token
        Slack.configure do |config|
          config.token = @conf.slack_token
        end
    end
    if @conf.smtp_server
        conf_ = @conf
        Mail.defaults do
          delivery_method :smtp, address: conf_.smtp_server, port: 587, :domain => conf_.smtp_domain, 
                          :enable_starttls_auto => true, :openssl_verify_mode => 'none'
        end
    end
end