class LogStash::Filters::CSV

The CSV filter takes an event field containing CSV data, parses it, and stores it as individual fields (can optionally specify the names). This filter can also parse data with any separator, not just commas.

Constants

CONVERTERS

Public Instance Methods

filter(event) click to toggle source
# File lib/logstash/filters/csv.rb, line 135
def filter(event)
  @logger.debug? && @logger.debug("Running csv filter", :event => event.to_hash)

  if (source = event.get(@source))
    begin

      values = CSV.parse_line(source, :col_sep => @separator, :quote_char => @quote_char)        

      if (@autodetect_column_names && @columns.empty?)
        @columns = values
        event.cancel
        return
      end

      if (@skip_header && (!@columns.empty?) && (@columns == values))
        event.cancel
        return
      end

      if(@skip_empty_rows && values.nil?)
        # applies tag to empty rows, users can cancel event referencing this tag in an 'if' conditional statement
        event.tag("_csvskippedemptyfield")
        return
      end

      values.each_index do |i|
        unless (@skip_empty_columns && (values[i].nil? || values[i].empty?))
          unless ignore_field?(i)
            field_name = @columns[i] || "column#{i + 1}"
            event.set(field_ref(field_name), transform(field_name, values[i]))
          end
        end
      end

      filter_matched(event)
    rescue => e
      event.tag("_csvparsefailure")
      @logger.warn("Error parsing csv", :field => @source, :source => source, :exception => e)
      return
    end
  end

  @logger.debug? && @logger.debug("Event after csv filter", :event => event.to_hash)
end
register() click to toggle source
# File lib/logstash/filters/csv.rb, line 113
def register
  # validate conversion types to be the valid ones.
  bad_types = @convert.values.select do |type|
    !CONVERTERS.has_key?(type.to_sym)
  end.uniq

  raise(LogStash::ConfigurationError, "Invalid conversion types: #{bad_types.join(', ')}") unless bad_types.empty?

  # @convert_symbols contains the symbolized types to avoid symbol conversion in the transform method
  @convert_symbols = @convert.inject({}){|result, (k, v)| result[k] = v.to_sym; result}

  # make sure @target is in the format [field name] if defined, i.e. surrounded by brackets
  @target = "[#{@target}]" if @target && !@target.start_with?("[")
  
  # if the zero byte character is entered in the config, set the value
  if (@quote_char == "\\x00")
    @quote_char = "\x00"
  end
  
  @logger.debug? && @logger.debug("CSV parsing options", :col_sep => @separator, :quote_char => @quote_char)
end

Private Instance Methods

field_ref(field_name) click to toggle source

construct the correct Event field reference for given field_name, taking into account @target @param field_name [String] the field name. @return [String] fully qualified Event field reference also taking into account @target prefix

# File lib/logstash/filters/csv.rb, line 185
def field_ref(field_name)
  if field_name.start_with?("[")
    "#{@target}#{field_name}"
  else
    "#{@target}[#{field_name}]"
  end
end
ignore_field?(index) click to toggle source
# File lib/logstash/filters/csv.rb, line 193
def ignore_field?(index)
  !@columns[index] && !@autogenerate_column_names
end
transform(field_name, value) click to toggle source
# File lib/logstash/filters/csv.rb, line 197
def transform(field_name, value)
  CONVERTERS[@convert_symbols[field_name]].call(value)
end