class LogStash::Filters::Webookup
Constants
- HTTP_OPTIONS
Public Instance Methods
filter(event)
click to toggle source
# File lib/logstash/filters/weblookup.rb, line 98 def filter(event) if destinations[0] == "srcdst" # ... do special sauce begin src = parse(event.get(fields[0]).to_s) dst = parse(event.get(fields[1]).to_s) srcdst = { :srcnet => src["netname"], :srchost => src["hostname"], :dstnet => dst["netname"], :dsthost => dst["hostname"] } event.set("srcdst", srcdst) event.get("[srcdst]").each {|k, v| event.set(k, v) } event.remove("[srcdst]") @logger.trace("processed: #{event.get(fields[0]).to_s} #{src} #{event.get(fields[1]).to_s} #{dst} #{srcdst}") rescue Exception => e @logger.error(" caught: #{e.message}") end else fields.each_with_index do |field, index| # @logger.info(event.get("["+field+"]")) begin json = parse(event.get(field).to_s) event.set("["+destinations[index]+"]", json) rescue Exception => e @logger.error(" caught: #{e.message}") end end end if @normalize replant(event, @newroot) end # filter_matched should go in the last line of our successful code filter_matched(event) end
register()
click to toggle source
# File lib/logstash/filters/weblookup.rb, line 56 def register if use_redis unless redis_path.to_s.strip.empty? @red = Redis.new(path: redis_path) else @red = Redis.new() end end # input fields and destinations @is_one_destination=false if destinations.size == 1 @logger.info("one destination found, it is #{destinations[0]}") @is_one_destination=true else if destinations.size != fields.size @logger.error("Configuration error, there must be an equal amount of destinations and fields, defaulting to using the field as a root for the new values. e.g. if the lookup is done on the value of [\"ClientIP\"] the destination will be [\"ClientIP\"][\"Key\"]") destinations=fields end # add case destination is empty to put the result in under the same field end # http connectionpool @uri = Addressable::URI.parse(url) @uri.merge!(HTTP_OPTIONS) #@http = Net::HTTP.new(uri.host, uri.port, HTTP_OPTIONS) @uri.port=80 if (@uri.port.nil? && @uri.scheme=="http") @uri.port=443 if (@uri.port.nil? && @uri.scheme=="https") # find the key where the value is <item>, otherwise just use the value @params = @uri.query_values(Hash) @params.each do |key, value| if value == "\<item\>" @ip=key @params.delete(key) logger.info("the ip key in the uri is #{@ip}") end end @connpool = ConnectionPool.new(size: 4, timeout: 180) { Net::HTTP.new(@uri.host, @uri.port) } end
Private Instance Methods
csv_loader(data)
click to toggle source
# File lib/logstash/filters/weblookup.rb, line 201 def csv_loader(data) data = CSV.read(data).inject(Hash.new) do |acc, v| acc[v[0]] = v[1] acc end get_map.merge!(data) end
find(item)
click to toggle source
# File lib/logstash/filters/weblookup.rb, line 143 def find(item) res = "{}" # Is item in list? (list is an optional array) #unless list.nil? # What if the list exists, but item is not on the list? # return list[item] #end # Is item in redis? unless @red.nil? res = @red.get(item) unless res.nil? return res end end # find the key where the value is <item>, otherwise just use the value current_uri = @uri current_uri.query_values = @params.merge({@ip => item}) #logger.info(@uri.to_s) @connpool.with do |conn| http_response = conn.request_get(current_uri) res = http_response.read_body if http_response.is_a?(Net::HTTPSuccess) if res.eql? "null" res = "{}" end #logger.info(res.to_s) unless @red.nil? @red.set(item, res) @red.expire(item,redis_expiry) end end return res end
json_loader(data)
click to toggle source
From github.com/angel9484/logstash-filter-lookup
# File lib/logstash/filters/weblookup.rb, line 197 def json_loader(data) get_map.merge!(JSON.parse(File.read(data))) end
parse(field)
click to toggle source
# File lib/logstash/filters/weblookup.rb, line 133 def parse(field) x = find(field) begin json = JSON.parse(x) rescue json = JSON.parse("{\"ip\": \""+field+"\"}") end # @logger.info("json parse option for field #{field} / #{json}") end
replant(event, newroot)
click to toggle source
for legacy
def normalize(event)
event.set("net", JSON.parse(net)) event.get("[records][properties]").each {|k, v| event.set(k, v) } event.remove("[records]") event.remove("[message]") return event
end
# File lib/logstash/filters/weblookup.rb, line 186 def replant(event, newroot) begin @logger.debug("event: #{event.get(newroot)}") event.get(newroot).each {|k, v| event.set(k, v) } event.remove(@roottostrip) rescue Exception => e @logger.info("event failed") end end
yml_loader(data)
click to toggle source
# File lib/logstash/filters/weblookup.rb, line 209 def yml_loader(data) get_map.merge!(YAML.load_file(data)) end