module GreenHat::Formatters

Log

Log

Log

Log

Log

Formatter

Log

Log

Log

Log

Log

Log

Log

Log

Log

Log

Log

Log

Log

Public Instance Methods

dmesg_split(raw) click to toggle source
# File lib/greenhat/thing/formatters/bracket_log.rb, line 34
def dmesg_split(raw)
  Shellwords.split(raw).each_with_object({}) do |x, h|
    key, value = x.split('=')
    next if value.nil?

    h[key] = value.numeric? ? value.to_f : value
    h[key] = 0.0 if h[key].numeric? && h[key].zero?
  end
end
flatten_hash(param, prefix = nil) click to toggle source
# File lib/greenhat/thing/formatters/api_json.rb, line 27
def flatten_hash(param, prefix = nil)
  param.each_pair.reduce({}) do |a, (k, v)|
    v.is_a?(Hash) ? a.merge(flatten_hash(v, "#{prefix}#{k}.")) : a.merge("#{prefix}#{k}".to_sym => v)
  end
end
format_api_json() click to toggle source

Formatters

Primarily for gitlab-rails/api_json.log

# File lib/greenhat/thing/formatters/api_json.rb, line 9
def format_api_json
  self.result = raw.map do |row|
    result = Oj.load row

    # Parsing Time
    format_json_traverse result

    flatten_hash(result).sort.to_h
  rescue StandardError => e
    # TODO: Background Logger?
    e.message
    LogBot.warn('JSON Parse', e.message)
    next
  end

  :ok
end
format_bracket_log() click to toggle source

Formatters for bracket logs (dmesg, sos)

# File lib/greenhat/thing/formatters/bracket_log.rb, line 6
def format_bracket_log
  self.result = raw.map do |row|
    next if row.empty? || row == "\n"

    result = {}
    time, output = row.split(']', 2)
    result[:time] = Time.parse time.split('[', 2).last.strip

    if output.include? ': '
      category, raw = output.split(': ', 2)
      result[:category] = category.strip

      result.merge! dmesg_split(raw) if raw.include? '='

      result[:message] = raw.strip
    else
      result[:message] = output
    end

    result
  rescue StandardError => e
    # TODO: Background logger
    LogBot.fatal('dmesg', "Unable to Parse, #{row}:#{e.message}")
  end

  self.result.compact!
end
format_clean_raw() click to toggle source

Remove Comments / Empty Lines

# File lib/greenhat/thing/formatters/clean_raw.rb, line 6
def format_clean_raw
  staging = raw.clone

  # Empty
  staging.reject!(&:empty?)

  # Commented
  staging.reject! { |x| x =~ /#.*$/ }

  self.result = if staging.empty?
                  raw
                else
                  staging
                end
end
format_colon_split_strip() click to toggle source

Formatters for single json blobs in entire file

# File lib/greenhat/thing/formatters/colon_split_strip.rb, line 6
def format_colon_split_strip
  self.result = raw.map do |row|
    row.split(':', 2).map(&:strip)
  end.to_h
end
format_dotenv() click to toggle source

Formatters for Dmesg

# File lib/greenhat/thing/formatters/dotenv.rb, line 6
def format_dotenv
  self.result = Dotenv::Parser.new(raw.join("\n")).call
end
format_free_m() click to toggle source

Get Split Memory Table

# File lib/greenhat/thing/formatters/free_m.rb, line 6
def format_free_m
  # Headers to Readable Symbol
  headers = raw.first.split(' ', 6).map(&:downcase).map do |x|
    x.gsub(/\s+/, '_').gsub(/[^0-9A-Za-z_]/, '')
  end.map(&:to_sym)

  # Add Kind
  headers.unshift(:kind)

  final = []

  # Put fields into a Hash based on Location/Key
  raw[1..].map(&:split).each do |row|
    result = {}
    row.each_with_index do |detail, i|
      result[headers[i]] = detail.split(':').first
    end
    final.push result
  end

  self.result = final
end
format_gitlab_status() click to toggle source

Remove Comments / Empty Lines

# File lib/greenhat/thing/formatters/gitlab_status.rb, line 6
def format_gitlab_status
  final = {}
  raw.each do |row|
    list = row.split('; ').map do |entry|
      status, service, pid_uptime = entry.split(': ')

      {
        status: status,
        name: service,
        pid_uptime: pid_uptime
      }
    end

    final[list.first.name] = list
  end

  self.result = final
end
format_gitlab_tail() click to toggle source

Gitlab Tail Formatter

# File lib/greenhat/thing/formatters/gitlab_ctl_tail.rb, line 8
def format_gitlab_tail
  # Revert to raw for cats
  self.kind = :raw

  output = {}
  current_log = nil

  raw.each do |line|
    next if line.blank?

    if line.include? '==>'
      current_log = /==> (.+?) <==/.match(line).captures.first
    else
      output[current_log] ||= []
      output[current_log].push line
    end
  end

  # Remove Empty Entries
  output.reject { |_k, v| v.empty? }

  # Root Dir
  root_dir = "#{$TMP}/#{name}"
  Dir.mkdir(root_dir)

  # Write Files / Create Things
  output.each do |k, v|
    file_name = k.gsub('/var/log/gitlab/', '')

    dir = "#{root_dir}/#{file_name.split('/').first}"
    Dir.mkdir(dir) unless File.exist?(dir)

    File.write("#{root_dir}/#{file_name}", v.join("\n"))

    # Thing Setup
    archive.things_create(file: "#{root_dir}/#{file_name}").setup
  end

  # Link
  self.result = raw
end
format_json() click to toggle source

Formatters Fallback to String

# File lib/greenhat/thing/formatters/json.rb, line 9
def format_json
  self.result = raw.map do |row|
    result = begin
      Oj.load row
    rescue EncodingError
      { message: row }
    end

    # Parsing Time
    format_json_traverse result

    result.sort.to_h
  rescue StandardError => e
    # TODO: Background Logger?
    e.message
    LogBot.warn('JSON Parse', e.message)
    next
  end

  :ok
end
format_json_shell() click to toggle source

Formatters

# File lib/greenhat/thing/formatters/json_shellwords.rb, line 8
def format_json_shell
  self.result = raw.map do |row|
    result = begin
      Oj.load row
    rescue EncodingError
      json_shellwords_fallback row
    end

    # Parsing Time
    format_json_traverse result

    result.sort.to_h
  rescue StandardError => e
    LogBot.warn('JSON Parse', e.message)
    next
  end

  :ok
end
format_json_time(result) click to toggle source

Check for Common Fields

# File lib/greenhat/thing/formatters/json.rb, line 44
def format_json_time(result)
  result.time = format_time_parse(result.time) if result.key? :time
  result.created_at = format_time_parse(result.created_at) if result.key? :created_at
  result.enqueued_at = format_time_parse(result.enqueued_at) if result.key? :enqueued_at
rescue StandardError => e
  LogBot.warn('JSON Time Parse', e.message)
  true
end
format_json_traverse(result) click to toggle source

Recursively Navigate

# File lib/greenhat/thing/formatters/json.rb, line 33
def format_json_traverse(result)
  format_json_time(result)

  result.each do |_key, value|
    next unless value.instance_of? Hash

    format_json_traverse(value)
  end
end
format_multiline_json() click to toggle source

Formatters for single json blobs in entire file

# File lib/greenhat/thing/formatters/multiline_json.rb, line 6
def format_multiline_json
  self.result = Oj.load raw.join
end
format_raw() click to toggle source
# File lib/greenhat/thing/formatters/raw.rb, line 13
def format_raw
  self.result = raw
end
format_shellwords() click to toggle source

Formatters Not Handled

# File lib/greenhat/thing/formatters/shellwords.rb, line 8
def format_shellwords
  self.result = raw.map do |row|
    result = Shellwords.split(row).each_with_object({}) do |x, h|
      key, value = x.split('=')
      next if value.nil?

      h[key] = value.numeric? ? value.to_f : value
    end
    result.time = Time.parse result.time if result.key? 'time'

    result
  end
end
format_syslog() click to toggle source

Formatters for bracket logs (dmesg, sos)

# File lib/greenhat/thing/formatters/syslog.rb, line 6
def format_syslog
  self.result = raw.map do |row|
    next if row.empty? || row == "\n"

    format_syslog_row(row)
  end

  result.compact!
end
format_syslog_row(row) click to toggle source

Split / Parse Time TODO: Better sys log parsing? Cannot find ready-made regex/ruby parser

# File lib/greenhat/thing/formatters/syslog.rb, line 18
def format_syslog_row(row)
  month, day, time, device, service, message = row.split(' ', 6)
  service.gsub!(':', '')
  pid = service.match(/\[(.*?)\]/)&.captures&.first

  {
    time: format_time_parse("#{month} #{day} #{time}"),
    device: device,
    service: service,
    message: message,
    pid: pid
  }

# Return everything incase of error
rescue StandardError => e
  LogBot.warn('SysLog Parse', e.message)
  {
    message: row
  }
end
format_table() click to toggle source

Formatters for Dmesg

# File lib/greenhat/thing/formatters/table.rb, line 6
def format_table
  # Headers to Readable Symbol
  headers = raw.first.split(' ', 6).map(&:downcase).map do |x|
    x.gsub(/\s+/, '_').gsub(/[^0-9A-Za-z_]/, '')
  end.map(&:to_sym)

  final = []

  # Put fields into a Hash based on Location/Key
  raw[1..].map(&:split).each do |row|
    result = {}
    row.each_with_index do |detail, i|
      result[headers[i]] = detail
    end
    final.push result
  end

  self.result = final
end
format_time_json() click to toggle source

Time Space JSON 2021-05-04_18:29:28.66542 {“timestamp”:“2021-05-04T18:29:28.665Z”,“pid”:3435539,“message”:“Use Ctrl-C to stop”}

# File lib/greenhat/thing/formatters/time_json.rb, line 9
def format_time_json
  self.result = raw.map do |row|
    time, msg = row.split(' ', 2)

    result =  Oj.load msg
    result.time = time

    result
  end
end
format_time_parse(time) click to toggle source

Handle Epoch Timestamps as well as string timestamps

# File lib/greenhat/thing/formatters/json.rb, line 54
def format_time_parse(time)
  if time.numeric?
    Time.at time
  else
    Time.parse time
  end
end
format_time_shellwords() click to toggle source

Formatters Not Handled

# File lib/greenhat/thing/formatters/time_shellwords.rb, line 8
def format_time_shellwords
  self.result = raw.map do |row|
    time, msg = row.split(' ', 2)

    result = Shellwords.split(msg).each_with_object({}) do |x, h|
      key, value = x.split('=')
      next if value.nil?

      h[key] = value.numeric? ? value.to_f : value
    end

    # Timestamp Parsing
    result.ts = Time.parse result.ts if result.key? 'ts'
    result.time = Time.parse time

    result
  end
end
format_time_space() click to toggle source

Formatters Not Handled

# File lib/greenhat/thing/formatters/time_space.rb, line 8
def format_time_space
  self.result = raw.map do |row|
    time, msg = row.split(' ', 2)

    {
      time: Time.parse(time),
      msg: msg
    }
  end
end
json_shellwords_fallback(row) click to toggle source
# File lib/greenhat/thing/formatters/json_shellwords.rb, line 28
def json_shellwords_fallback(row)
  result = Shellwords.split(row).each_with_object({}) do |x, h|
    key, value = x.split('=')
    next if value.nil?

    h[key] = value.numeric? ? value.to_f : value
  end
  result.time = Time.parse result.time if result.key? 'time'

  result
end
log_format() click to toggle source
# File lib/greenhat/thing/formatters/format.rb, line 4
def log_format
  self.result = send(SuperLog.type?(path)) if log_type
end
log_type() click to toggle source
# File lib/greenhat/thing/formatters/format.rb, line 8
def log_type
  SuperLog.type?(path)
end
mia() click to toggle source

Formatters Not Handled

# File lib/greenhat/thing/formatters/raw.rb, line 8
def mia
  # TODO: Background Logger
  LogBot.warn('Log Format', "No Formatter for #{name}::#{path}")
end