class LogStash::Filters::Mutate

The mutate filter allows you to perform general mutations on fields. You can rename, replace, and modify fields in your events.

Constants

CONVERT_PREFIX
FALSE_REGEX
TRUE_REGEX

Public Instance Methods

filter(event) click to toggle source
# File lib/logstash/filters/mutate.rb, line 250
def filter(event)
  coerce(event) if @coerce
  rename(event) if @rename
  update(event) if @update
  replace(event) if @replace
  convert(event) if @convert
  gsub(event) if @gsub
  uppercase(event) if @uppercase
  capitalize(event) if @capitalize
  lowercase(event) if @lowercase
  strip(event) if @strip
  split(event) if @split
  join(event) if @join
  merge(event) if @merge
  copy(event) if @copy

  filter_matched(event)
rescue => ex
  meta = { :exception => ex.message }
  meta[:backtrace] = ex.backtrace if logger.debug?
  logger.warn('Exception caught while applying mutate filter', meta)
  event.tag(@tag_on_failure)
end
register() click to toggle source
# File lib/logstash/filters/mutate.rb, line 217
def register
  valid_conversions = %w(string integer float boolean integer_eu float_eu )
  # TODO(sissel): Validate conversion requests if provided.
  @convert.nil? or @convert.each do |field, type|
    if !valid_conversions.include?(type)
      raise LogStash::ConfigurationError, I18n.t(
        "logstash.runner.configuration.invalid_plugin_register",
        :plugin => "filter",
        :type => "mutate",
        :error => "Invalid conversion type '#{type}', expected one of '#{valid_conversions.join(',')}'"
      )
    end
  end

  @gsub_parsed = []
  @gsub.nil? or @gsub.each_slice(3) do |field, needle, replacement|
    if [field, needle, replacement].any? {|n| n.nil?}
      raise LogStash::ConfigurationError, I18n.t(
        "logstash.runner.configuration.invalid_plugin_register",
        :plugin => "filter",
        :type => "mutate",
        :error => "Invalid gsub configuration #{[field, needle, replacement]}. gsub requires 3 non-nil elements per config entry"
      )
    end

    @gsub_parsed << {
      :field        => field,
      :needle       => (needle.index("%{").nil?? Regexp.new(needle): needle),
      :replacement  => replacement
    }
  end
end

Private Instance Methods

capitalize(event) click to toggle source
# File lib/logstash/filters/mutate.rb, line 455
def capitalize(event)
  #see comments for #uppercase
  @capitalize.each do |field|
    original = event.get(field)
    next if original.nil?
    result = case original
    when Array
      original.map! do |elem|
        (elem.is_a?(String) ? elem.capitalize : elem)
      end
    when String
      original.capitalize
    else
      @logger.debug? && @logger.debug("Can't capitalize something that isn't a string", :field => field, :value => original)
      original
    end
    event.set(field, result)
  end
end
cnv_replace_eu(value) click to toggle source

When given a String, returns a new String whose contents have been converted from EU-style comma-decimals and dot-separators to US-style dot-decimals and comma-separators.

For all other values, returns value unmodified.

# File lib/logstash/filters/mutate.rb, line 371
def cnv_replace_eu(value)
  return value if !value.is_a?(String)
  value.tr(",.", ".,")
end
coerce(event) click to toggle source
# File lib/logstash/filters/mutate.rb, line 276
def coerce(event)
  @coerce.each do |field, default_value|
    next unless event.include?(field) && event.get(field)==nil
    event.set(field, event.sprintf(default_value))
  end
end
convert(event) click to toggle source
# File lib/logstash/filters/mutate.rb, line 305
def convert(event)
  @convert.each do |field, type|
    next unless event.include?(field)
    original = event.get(field)
    # calls convert_{string,integer,float,boolean} depending on type requested.
    converter = method(CONVERT_PREFIX + type)

    case original
    when Hash
      @logger.debug? && @logger.debug("I don't know how to type convert a hash, skipping", :field => field, :value => original)
    when Array
      event.set(field, original.map { |v| v.nil? ? v : converter.call(v) })
    when NilClass
      # ignore
    else
      event.set(field, converter.call(original))
    end
  end
end
convert_boolean(value) click to toggle source
# File lib/logstash/filters/mutate.rb, line 336
def convert_boolean(value)
  return true if value.to_s =~ TRUE_REGEX
  return false if value.to_s.empty? || value.to_s =~ FALSE_REGEX
  @logger.warn("Failed to convert #{value} into boolean.")
  value
end
convert_float(value) click to toggle source
# File lib/logstash/filters/mutate.rb, line 350
def convert_float(value)
  return 1.0 if value == true
  return 0.0 if value == false
  value = value.delete(",") if value.kind_of?(String)
  value.to_f
end
convert_float_eu(value) click to toggle source
# File lib/logstash/filters/mutate.rb, line 362
def convert_float_eu(value)
  us_value = cnv_replace_eu(value)
  convert_float(us_value)
end
convert_integer(value) click to toggle source
# File lib/logstash/filters/mutate.rb, line 343
def convert_integer(value)
  return 1 if value == true
  return 0 if value == false
  return value.to_i if !value.is_a?(String)
  value.tr(",", "").to_i
end
convert_integer_eu(value) click to toggle source
# File lib/logstash/filters/mutate.rb, line 357
def convert_integer_eu(value)
  us_value = cnv_replace_eu(value)
  convert_integer(us_value)
end
convert_string(value) click to toggle source
# File lib/logstash/filters/mutate.rb, line 325
def convert_string(value)
  # since this is a filter and all inputs should be already UTF-8
  # we wont check valid_encoding? but just force UTF-8 for
  # the Fixnum#to_s case which always result in US-ASCII
  # also not that force_encoding checks current encoding against the
  # target encoding and only change if necessary, so calling
  # valid_encoding? is redundant
  # see https://twitter.com/jordansissel/status/444613207143903232
  value.to_s.force_encoding(Encoding::UTF_8)
end
copy(event) click to toggle source
# File lib/logstash/filters/mutate.rb, line 537
def copy(event)
  @copy.each do |src_field, dest_field|
    original = event.get(src_field)
    next if original.nil?
    event.set(dest_field,LogStash::Util.deep_clone(original))
  end
end
gsub(event) click to toggle source
# File lib/logstash/filters/mutate.rb, line 376
def gsub(event)
  @gsub_parsed.each do |config|
    field = config[:field]
    needle = config[:needle]
    replacement = config[:replacement]

    value = event.get(field)
    case value
    when Array
      result = value.map do |v|
        if v.is_a?(String)
          gsub_dynamic_fields(event, v, needle, replacement)
        else
          @logger.warn("gsub mutation is only applicable for strings and arrays of strings, skipping", :field => field, :value => v)
          v
        end
      end
      event.set(field, result)
    when String
      event.set(field, gsub_dynamic_fields(event, value, needle, replacement))
    else
      @logger.debug? && @logger.debug("gsub mutation is only applicable for strings and arrays of strings, skipping", :field => field, :value => event.get(field))
    end
  end
end
gsub_dynamic_fields(event, original, needle, replacement) click to toggle source
# File lib/logstash/filters/mutate.rb, line 402
def gsub_dynamic_fields(event, original, needle, replacement)
  if needle.is_a?(Regexp)
    original.gsub(needle, event.sprintf(replacement))
  else
    # we need to replace any dynamic fields
    original.gsub(Regexp.new(event.sprintf(needle)), event.sprintf(replacement))
  end
end
join(event) click to toggle source
# File lib/logstash/filters/mutate.rb, line 486
def join(event)
  @join.each do |field, separator|
    value = event.get(field)
    if value.is_a?(Array)
      event.set(field, value.join(separator))
    end
  end
end
lowercase(event) click to toggle source
# File lib/logstash/filters/mutate.rb, line 435
def lowercase(event)
  #see comments for #uppercase
  @lowercase.each do |field|
    original = event.get(field)
    next if original.nil?
    result = case original
    when Array
      original.map! do |elem|
        (elem.is_a?(String) ? elem.downcase : elem)
      end
    when String
      original.downcase
    else
      @logger.debug? && @logger.debug("Can't lowercase something that isn't a string", :field => field, :value => original)
      original
    end
    event.set(field, result)
  end
end
merge(event) click to toggle source
# File lib/logstash/filters/mutate.rb, line 507
def merge(event)
  @merge.each do |dest_field, added_fields|
    # When multiple calls, added_field is an array

    dest_field_value = event.get(dest_field)

    Array(added_fields).each do |added_field|
      added_field_value = event.get(added_field)

      if dest_field_value.is_a?(Hash) ^ added_field_value.is_a?(Hash)
        @logger.error("Not possible to merge an array and a hash: ", :dest_field => dest_field, :added_field => added_field )
        next
      end

      # No need to test the other
      if dest_field_value.is_a?(Hash)
        # do not use event[dest_field].update because the returned object from event[dest_field]
        # can/will be a copy of the actual event data and directly updating it will not update
        # the Event internal data. The updated value must be reassigned in the Event.
        event.set(dest_field, dest_field_value.update(added_field_value))
      else
        # do not use event[dest_field].concat because the returned object from event[dest_field]
        # can/will be a copy of the actual event data and directly updating it will not update
        # the Event internal data. The updated value must be reassigned in the Event.
        event.set(dest_field, Array(dest_field_value).concat(Array(added_field_value)))
      end
    end
  end
end
rename(event) click to toggle source
# File lib/logstash/filters/mutate.rb, line 283
def rename(event)
  @rename.each do |old, new|
    old = event.sprintf(old)
    new = event.sprintf(new)
    next unless event.include?(old)
    event.set(new, event.remove(old))
  end
end
replace(event) click to toggle source
# File lib/logstash/filters/mutate.rb, line 299
def replace(event)
  @replace.each do |field, newvalue|
    event.set(field, event.sprintf(newvalue))
  end
end
split(event) click to toggle source
# File lib/logstash/filters/mutate.rb, line 475
def split(event)
  @split.each do |field, separator|
    value = event.get(field)
    if value.is_a?(String)
      event.set(field, value.split(separator))
    else
      @logger.debug? && @logger.debug("Can't split something that isn't a string", :field => field, :value => event.get(field))
    end
  end
end
strip(event) click to toggle source
# File lib/logstash/filters/mutate.rb, line 495
def strip(event)
  @strip.each do |field|
    value = event.get(field)
    case value
    when Array
      event.set(field, value.map{|s| s.strip })
    when String
      event.set(field, value.strip)
    end
  end
end
update(event) click to toggle source
# File lib/logstash/filters/mutate.rb, line 292
def update(event)
  @update.each do |field, newvalue|
    next unless event.include?(field)
    event.set(field, event.sprintf(newvalue))
  end
end
uppercase(event) click to toggle source
# File lib/logstash/filters/mutate.rb, line 411
def uppercase(event)
  @uppercase.each do |field|
    original = event.get(field)
    next if original.nil?
    # in certain cases JRuby returns a proxy wrapper of the event[field] value
    # therefore we can't assume that we are modifying the actual value behind
    # the key so read, modify and overwrite
    result = case original
      when Array
        # can't map upcase! as it replaces an already upcase value with nil
        # ["ABCDEF"].map(&:upcase!) => [nil]
        original.map do |elem|
          (elem.is_a?(String) ? elem.upcase : elem)
        end
      when String
        original.upcase
      else
        @logger.debug? && @logger.debug("Can't uppercase something that isn't a string", :field => field, :value => original)
        original
      end
    event.set(field, result)
  end
end