module GreenHat::Formatters
Log
Log
Log
Log
Log
Formatter
Log
Log
Log
Log
Log
Log
Log
Log
Log
Log
Log
Log
Log
Public Instance Methods
# File lib/greenhat/thing/formatters/bracket_log.rb, line 34 def dmesg_split(raw) Shellwords.split(raw).each_with_object({}) do |x, h| key, value = x.split('=') next if value.nil? h[key] = value.numeric? ? value.to_f : value h[key] = 0.0 if h[key].numeric? && h[key].zero? end end
# File lib/greenhat/thing/formatters/api_json.rb, line 27 def flatten_hash(param, prefix = nil) param.each_pair.reduce({}) do |a, (k, v)| v.is_a?(Hash) ? a.merge(flatten_hash(v, "#{prefix}#{k}.")) : a.merge("#{prefix}#{k}".to_sym => v) end end
¶ ↑
¶ ↑
Primarily for gitlab-rails/api_json.log
# File lib/greenhat/thing/formatters/api_json.rb, line 9 def format_api_json self.result = raw.map do |row| result = Oj.load row # Parsing Time format_json_traverse result flatten_hash(result).sort.to_h rescue StandardError => e # TODO: Background Logger? e.message LogBot.warn('JSON Parse', e.message) next end :ok end
Formatters
for bracket logs (dmesg, sos)
# File lib/greenhat/thing/formatters/bracket_log.rb, line 6 def format_bracket_log self.result = raw.map do |row| next if row.empty? || row == "\n" result = {} time, output = row.split(']', 2) result[:time] = Time.parse time.split('[', 2).last.strip if output.include? ': ' category, raw = output.split(': ', 2) result[:category] = category.strip result.merge! dmesg_split(raw) if raw.include? '=' result[:message] = raw.strip else result[:message] = output end result rescue StandardError => e # TODO: Background logger LogBot.fatal('dmesg', "Unable to Parse, #{row}:#{e.message}") end self.result.compact! end
Remove Comments / Empty Lines
# File lib/greenhat/thing/formatters/clean_raw.rb, line 6 def format_clean_raw staging = raw.clone # Empty staging.reject!(&:empty?) # Commented staging.reject! { |x| x =~ /#.*$/ } self.result = if staging.empty? raw else staging end end
Formatters
for single json blobs in entire file
# File lib/greenhat/thing/formatters/colon_split_strip.rb, line 6 def format_colon_split_strip self.result = raw.map do |row| row.split(':', 2).map(&:strip) end.to_h end
Formatters
for Dmesg
# File lib/greenhat/thing/formatters/dotenv.rb, line 6 def format_dotenv self.result = Dotenv::Parser.new(raw.join("\n")).call end
Get Split Memory
Table
# File lib/greenhat/thing/formatters/free_m.rb, line 6 def format_free_m # Headers to Readable Symbol headers = raw.first.split(' ', 6).map(&:downcase).map do |x| x.gsub(/\s+/, '_').gsub(/[^0-9A-Za-z_]/, '') end.map(&:to_sym) # Add Kind headers.unshift(:kind) final = [] # Put fields into a Hash based on Location/Key raw[1..].map(&:split).each do |row| result = {} row.each_with_index do |detail, i| result[headers[i]] = detail.split(':').first end final.push result end self.result = final end
Remove Comments / Empty Lines
# File lib/greenhat/thing/formatters/gitlab_status.rb, line 6 def format_gitlab_status final = {} raw.each do |row| list = row.split('; ').map do |entry| status, service, pid_uptime = entry.split(': ') { status: status, name: service, pid_uptime: pid_uptime } end final[list.first.name] = list end self.result = final end
¶ ↑
Gitlab Tail Formatter
¶ ↑
# File lib/greenhat/thing/formatters/gitlab_ctl_tail.rb, line 8 def format_gitlab_tail # Revert to raw for cats self.kind = :raw output = {} current_log = nil raw.each do |line| next if line.blank? if line.include? '==>' current_log = /==> (.+?) <==/.match(line).captures.first else output[current_log] ||= [] output[current_log].push line end end # Remove Empty Entries output.reject { |_k, v| v.empty? } # Root Dir root_dir = "#{$TMP}/#{name}" Dir.mkdir(root_dir) # Write Files / Create Things output.each do |k, v| file_name = k.gsub('/var/log/gitlab/', '') dir = "#{root_dir}/#{file_name.split('/').first}" Dir.mkdir(dir) unless File.exist?(dir) File.write("#{root_dir}/#{file_name}", v.join("\n")) # Thing Setup archive.things_create(file: "#{root_dir}/#{file_name}").setup end # Link self.result = raw end
¶ ↑
Formatters
Fallback to String
¶ ↑
# File lib/greenhat/thing/formatters/json.rb, line 9 def format_json self.result = raw.map do |row| result = begin Oj.load row rescue EncodingError { message: row } end # Parsing Time format_json_traverse result result.sort.to_h rescue StandardError => e # TODO: Background Logger? e.message LogBot.warn('JSON Parse', e.message) next end :ok end
¶ ↑
¶ ↑
# File lib/greenhat/thing/formatters/json_shellwords.rb, line 8 def format_json_shell self.result = raw.map do |row| result = begin Oj.load row rescue EncodingError json_shellwords_fallback row end # Parsing Time format_json_traverse result result.sort.to_h rescue StandardError => e LogBot.warn('JSON Parse', e.message) next end :ok end
Check for Common Fields
# File lib/greenhat/thing/formatters/json.rb, line 44 def format_json_time(result) result.time = format_time_parse(result.time) if result.key? :time result.created_at = format_time_parse(result.created_at) if result.key? :created_at result.enqueued_at = format_time_parse(result.enqueued_at) if result.key? :enqueued_at rescue StandardError => e LogBot.warn('JSON Time Parse', e.message) true end
Recursively Navigate
# File lib/greenhat/thing/formatters/json.rb, line 33 def format_json_traverse(result) format_json_time(result) result.each do |_key, value| next unless value.instance_of? Hash format_json_traverse(value) end end
Formatters
for single json blobs in entire file
# File lib/greenhat/thing/formatters/multiline_json.rb, line 6 def format_multiline_json self.result = Oj.load raw.join end
# File lib/greenhat/thing/formatters/raw.rb, line 13 def format_raw self.result = raw end
¶ ↑
Formatters
Not Handled
¶ ↑
# File lib/greenhat/thing/formatters/shellwords.rb, line 8 def format_shellwords self.result = raw.map do |row| result = Shellwords.split(row).each_with_object({}) do |x, h| key, value = x.split('=') next if value.nil? h[key] = value.numeric? ? value.to_f : value end result.time = Time.parse result.time if result.key? 'time' result end end
Formatters
for bracket logs (dmesg, sos)
# File lib/greenhat/thing/formatters/syslog.rb, line 6 def format_syslog self.result = raw.map do |row| next if row.empty? || row == "\n" format_syslog_row(row) end result.compact! end
Split / Parse Time TODO: Better sys log parsing? Cannot find ready-made regex/ruby parser
# File lib/greenhat/thing/formatters/syslog.rb, line 18 def format_syslog_row(row) month, day, time, device, service, message = row.split(' ', 6) service.gsub!(':', '') pid = service.match(/\[(.*?)\]/)&.captures&.first { time: format_time_parse("#{month} #{day} #{time}"), device: device, service: service, message: message, pid: pid } # Return everything incase of error rescue StandardError => e LogBot.warn('SysLog Parse', e.message) { message: row } end
Formatters
for Dmesg
# File lib/greenhat/thing/formatters/table.rb, line 6 def format_table # Headers to Readable Symbol headers = raw.first.split(' ', 6).map(&:downcase).map do |x| x.gsub(/\s+/, '_').gsub(/[^0-9A-Za-z_]/, '') end.map(&:to_sym) final = [] # Put fields into a Hash based on Location/Key raw[1..].map(&:split).each do |row| result = {} row.each_with_index do |detail, i| result[headers[i]] = detail end final.push result end self.result = final end
¶ ↑
Time Space JSON 2021-05-04_18:29:28.66542 {“timestamp”:“2021-05-04T18:29:28.665Z”,“pid”:3435539,“message”:“Use Ctrl-C to stop”}
¶ ↑
# File lib/greenhat/thing/formatters/time_json.rb, line 9 def format_time_json self.result = raw.map do |row| time, msg = row.split(' ', 2) result = Oj.load msg result.time = time result end end
Handle Epoch Timestamps as well as string timestamps
# File lib/greenhat/thing/formatters/json.rb, line 54 def format_time_parse(time) if time.numeric? Time.at time else Time.parse time end end
¶ ↑
Formatters
Not Handled
¶ ↑
# File lib/greenhat/thing/formatters/time_shellwords.rb, line 8 def format_time_shellwords self.result = raw.map do |row| time, msg = row.split(' ', 2) result = Shellwords.split(msg).each_with_object({}) do |x, h| key, value = x.split('=') next if value.nil? h[key] = value.numeric? ? value.to_f : value end # Timestamp Parsing result.ts = Time.parse result.ts if result.key? 'ts' result.time = Time.parse time result end end
¶ ↑
Formatters
Not Handled
¶ ↑
# File lib/greenhat/thing/formatters/time_space.rb, line 8 def format_time_space self.result = raw.map do |row| time, msg = row.split(' ', 2) { time: Time.parse(time), msg: msg } end end
# File lib/greenhat/thing/formatters/json_shellwords.rb, line 28 def json_shellwords_fallback(row) result = Shellwords.split(row).each_with_object({}) do |x, h| key, value = x.split('=') next if value.nil? h[key] = value.numeric? ? value.to_f : value end result.time = Time.parse result.time if result.key? 'time' result end
# File lib/greenhat/thing/formatters/format.rb, line 4 def log_format self.result = send(SuperLog.type?(path)) if log_type end
# File lib/greenhat/thing/formatters/format.rb, line 8 def log_type SuperLog.type?(path) end
¶ ↑
Formatters
Not Handled
¶ ↑
# File lib/greenhat/thing/formatters/raw.rb, line 8 def mia # TODO: Background Logger LogBot.warn('Log Format', "No Formatter for #{name}::#{path}") end