class LogStash::Filters::CSV
The CSV
filter takes an event field containing CSV
data, parses it, and stores it as individual fields (can optionally specify the names). This filter can also parse data with any separator, not just commas.
Constants
- CONVERTERS
Public Instance Methods
filter(event)
click to toggle source
# File lib/logstash/filters/csv.rb, line 135 def filter(event) @logger.debug? && @logger.debug("Running csv filter", :event => event.to_hash) if (source = event.get(@source)) begin values = CSV.parse_line(source, :col_sep => @separator, :quote_char => @quote_char) if (@autodetect_column_names && @columns.empty?) @columns = values event.cancel return end if (@skip_header && (!@columns.empty?) && (@columns == values)) event.cancel return end if(@skip_empty_rows && values.nil?) # applies tag to empty rows, users can cancel event referencing this tag in an 'if' conditional statement event.tag("_csvskippedemptyfield") return end values.each_index do |i| unless (@skip_empty_columns && (values[i].nil? || values[i].empty?)) unless ignore_field?(i) field_name = @columns[i] || "column#{i + 1}" event.set(field_ref(field_name), transform(field_name, values[i])) end end end filter_matched(event) rescue => e event.tag("_csvparsefailure") @logger.warn("Error parsing csv", :field => @source, :source => source, :exception => e) return end end @logger.debug? && @logger.debug("Event after csv filter", :event => event.to_hash) end
register()
click to toggle source
# File lib/logstash/filters/csv.rb, line 113 def register # validate conversion types to be the valid ones. bad_types = @convert.values.select do |type| !CONVERTERS.has_key?(type.to_sym) end.uniq raise(LogStash::ConfigurationError, "Invalid conversion types: #{bad_types.join(', ')}") unless bad_types.empty? # @convert_symbols contains the symbolized types to avoid symbol conversion in the transform method @convert_symbols = @convert.inject({}){|result, (k, v)| result[k] = v.to_sym; result} # make sure @target is in the format [field name] if defined, i.e. surrounded by brackets @target = "[#{@target}]" if @target && !@target.start_with?("[") # if the zero byte character is entered in the config, set the value if (@quote_char == "\\x00") @quote_char = "\x00" end @logger.debug? && @logger.debug("CSV parsing options", :col_sep => @separator, :quote_char => @quote_char) end
Private Instance Methods
field_ref(field_name)
click to toggle source
construct the correct Event field reference for given field_name, taking into account @target @param field_name [String] the field name. @return [String] fully qualified Event field reference also taking into account @target prefix
# File lib/logstash/filters/csv.rb, line 185 def field_ref(field_name) if field_name.start_with?("[") "#{@target}#{field_name}" else "#{@target}[#{field_name}]" end end
ignore_field?(index)
click to toggle source
# File lib/logstash/filters/csv.rb, line 193 def ignore_field?(index) !@columns[index] && !@autogenerate_column_names end
transform(field_name, value)
click to toggle source
# File lib/logstash/filters/csv.rb, line 197 def transform(field_name, value) CONVERTERS[@convert_symbols[field_name]].call(value) end