class LogStash::Filters::GeoIP
The GeoIP
filter adds information about the geographical location of IP addresses, based on data from the Maxmind database.
Starting with version 1.3.0 of Logstash, a `[geoip]` field is created if the GeoIP
lookup returns a latitude and longitude. The field is stored in geojson.org/geojson-spec.html[GeoJSON] format. Additionally, the default Elasticsearch template provided with the <<plugins-outputs-elasticsearch,`elasticsearch` output>> maps the `[geoip]` field to an www.elastic.co/guide/en/elasticsearch/reference/1.7/mapping-geo-point-type.html#_mapping_options[Elasticsearch geo_point].
As this field is a `geo_point` and it is still valid GeoJSON, you get the awesomeness of Elasticsearch's geospatial query, facet and filter functions and the flexibility of having GeoJSON for all other applications (like Kibana's map visualization).
Logstash releases ship with the GeoLiteCity database made available from Maxmind with a CCA-ShareAlike 3.0 license. For more details on GeoLite, see <www.maxmind.com/en/geolite>.
Constants
- LOOKUP_CACHES
Map of lookup caches, keyed by geoip_type
- LOOKUP_CACHE_INIT_MUTEX
Attributes
Public Instance Methods
# File lib/logstash/filters/geoip.rb, line 135 def apply_geodata(geo_data_hash, event) # don't do anything more if the lookup result is nil? return false if geo_data_hash.nil? # only set the event[@target] if the lookup result is not nil: BWC event[@target] = {} if event[@target].nil? # don't do anything more if the lookup result is empty? return false if geo_data_hash.empty? geo_data_hash.each do |key, value| if @no_fields || @fields.include?(key) # can't dup numerics event["[#{@target}][#{key}]"] = value.is_a?(Numeric) ? value : value.dup end end # geo_data_hash.each true end
# File lib/logstash/filters/geoip.rb, line 212 def ensure_database! # Use thread-local access to GeoIP. The Ruby GeoIP module forces a mutex # around access to the database, which can be overcome with :pread. # Unfortunately, :pread requires the io-extra gem, with C extensions that # aren't supported on JRuby. If / when :pread becomes available, we can stop # needing thread-local access. Thread.current[threadkey] ||= ::GeoIP.new(@database) end
# File lib/logstash/filters/geoip.rb, line 128 def filter(event) geo_data_hash = get_geo_data(event) if apply_geodata(geo_data_hash, event) filter_matched(event) end end
# File lib/logstash/filters/geoip.rb, line 151 def get_geo_data(event) # pure function, must control return value result = {} ip = event[@source] ip = ip.first if ip.is_a? Array return nil if ip.nil? begin result = get_geo_data_for_ip(ip) rescue SocketError => e @logger.error("IP Field contained invalid IP address or hostname", :field => @source, :event => event) rescue StandardError => e @logger.error("Unknown error while looking up GeoIP data", :exception => e, :field => @source, :event => event) end result end
# File lib/logstash/filters/geoip.rb, line 167 def get_geo_data_for_ip(ip) ensure_database! if (cached = lookup_cache[ip]) cached else geo_data = Thread.current[threadkey].send(@geoip_type, ip) converted = prepare_geodata_for_cache(geo_data) lookup_cache[ip] = converted converted end end
# File lib/logstash/filters/geoip.rb, line 179 def prepare_geodata_for_cache(geo_data) # GeoIP returns a nil or a Struct subclass return nil if !geo_data.respond_to?(:each_pair) #lets just do this once before caching result = {} geo_data.each_pair do |k, v| next if v.nil? || k == :request if v.is_a?(String) next if v.empty? # Some strings from GeoIP don't have the correct encoding... result[k.to_s] = case v.encoding # I have found strings coming from GeoIP that are ASCII-8BIT are actually # ISO-8859-1... when Encoding::ASCII_8BIT v.force_encoding(Encoding::ISO_8859_1).encode(Encoding::UTF_8) when Encoding::ISO_8859_1, Encoding::US_ASCII v.encode(Encoding::UTF_8) else v end else result[k.to_s] = v end end lat, lng = result.values_at("latitude", "longitude") if lat && lng result["location"] = [ lng.to_f, lat.to_f ] end result end
# File lib/logstash/filters/geoip.rb, line 88 def register require "geoip" if @database.nil? @database = ::Dir.glob(::File.join(::File.expand_path("../../../vendor/", ::File.dirname(__FILE__)),"GeoLiteCity*.dat")).first if !File.exists?(@database) raise "You must specify 'database => ...' in your geoip filter (I looked for '#{@database}'" end end @logger.info("Using geoip database", :path => @database) # For the purpose of initializing this filter, geoip is initialized here but # not set as a global. The geoip module imposes a mutex, so the filter needs # to re-initialize this later in the filter() thread, and save that access # as a thread-local variable. geoip_initialize = ::GeoIP.new(@database) @geoip_type = case geoip_initialize.database_type when GeoIP::GEOIP_CITY_EDITION_REV0, GeoIP::GEOIP_CITY_EDITION_REV1 :city when GeoIP::GEOIP_COUNTRY_EDITION :country when GeoIP::GEOIP_ASNUM_EDITION :asn when GeoIP::GEOIP_ISP_EDITION, GeoIP::GEOIP_ORG_EDITION :isp else raise RuntimeException.new "This GeoIP database is not currently supported" end @threadkey = "geoip-#{self.object_id}" # This is wrapped in a mutex to make sure the initialization behavior of LOOKUP_CACHES (see def above) doesn't create a dupe LOOKUP_CACHE_INIT_MUTEX.synchronize do self.lookup_cache = LOOKUP_CACHES[@geoip_type] ||= LruRedux::ThreadSafeCache.new(1000) end @no_fields = @fields.nil? || @fields.empty? end