class LogStash::Outputs::Graphite

This output allows you to pull metrics from your logs and ship them to Graphite. Graphite is an open source tool for storing and graphing metrics.

An example use case: Some applications emit aggregated stats in the logs every 10 seconds. Using the grok filter and this output, it is possible to capture the metric values from the logs and emit them to Graphite.

Constants

DEFAULT_METRICS_FORMAT
EXCLUDE_ALWAYS
METRIC_PLACEHOLDER

Public Instance Methods

connect() click to toggle source
# File lib/logstash/outputs/graphite.rb, line 92
def connect
  # TODO(sissel): Test error cases. Catch exceptions. Find fortune and glory. Retire to yak farm.
  begin
    @socket = TCPSocket.new(@host, @port)
  rescue Errno::ECONNREFUSED => e
    @logger.warn("Connection refused to graphite server, sleeping...", :host => @host, :port => @port)
    sleep(@reconnect_interval)
    retry
  end
end
construct_metric_name(event, metric) click to toggle source
# File lib/logstash/outputs/graphite.rb, line 103
def construct_metric_name(event, metric)
  if @metrics_format
    sprinted = event.sprintf(@metrics_format)
    return sprinted.gsub(METRIC_PLACEHOLDER, metric)
  end

  metric
end
receive(event) click to toggle source
# File lib/logstash/outputs/graphite.rb, line 112
def receive(event)
  # Graphite message format: metric value timestamp\n

  # compact to remove nil messages which produces useless \n
  messages = (
    @fields_are_metrics \
      ? messages_from_event_fields(event, @include_metrics, @exclude_metrics)
      : messages_from_event_metrics(event, @metrics)
  ).compact

  if messages.empty?
    @logger.debug? && @logger.debug("Message is empty, not sending anything to Graphite", :messages => messages, :host => @host, :port => @port)
  else
    message = messages.join("\n")
    @logger.debug? && @logger.debug("Sending carbon messages", :messages => messages, :host => @host, :port => @port)

    # Catch exceptions like ECONNRESET and friends, reconnect on failure.
    # TODO(sissel): Test error cases. Catch exceptions. Find fortune and glory.
    begin
      @socket.puts(message)
    rescue Errno::EPIPE, Errno::ECONNRESET, IOError => e
      @logger.warn("Connection to graphite server died", :exception => e.class, :message => e.message, :host => @host, :port => @port)
      sleep(@reconnect_interval)
      connect
      retry if @resend_on_failure
    rescue => e
      @logger.warn("Failed to send data", :exception => e.class, :message => e.message, :host => @host, :port => @port)
      sleep(@reconnect_interval)
      connect
      retry if @resend_on_failure
    end
  end
end
register() click to toggle source
# File lib/logstash/outputs/graphite.rb, line 79
def register
  @include_metrics.collect!{|regexp| Regexp.new(regexp)}
  @exclude_metrics.collect!{|regexp| Regexp.new(regexp)}

  if @metrics_format && !@metrics_format.include?(METRIC_PLACEHOLDER)
    @logger.warn("metrics_format does not include placeholder #{METRIC_PLACEHOLDER} .. falling back to default format: #{DEFAULT_METRICS_FORMAT.inspect}")

    @metrics_format = DEFAULT_METRICS_FORMAT
  end

  connect
end

Private Instance Methods

dotify(hash, prefix = nil) click to toggle source

Take a nested ruby hash of the form {:a => {:b => 2}, c: => 3} and turn it into a hash of the form { “a.b” => 2, “c” => 3}

# File lib/logstash/outputs/graphite.rb, line 195
def dotify(hash, prefix = nil)
  hash.reduce({}) do |acc, kv|
    k, v = kv
    pk = prefix ? "#{prefix}#{@nested_object_separator}#{k}" : k.to_s
    if v.is_a?(Hash)
      acc.merge!(dotify(v, pk))
    elsif v.is_a?(Array)
      # There's no right answer here, so we do nothing
      @logger.warn("Array values not supported for graphite metrics! Ignoring #{hash} @ #{prefix}")
    else
      acc[pk] = v
    end
    acc
  end
end
event_timestamp(event) click to toggle source
# File lib/logstash/outputs/graphite.rb, line 174
def event_timestamp(event)
  event.get(@timestamp_field).to_i
end
messages_from_event_fields(event, include_metrics, exclude_metrics) click to toggle source
# File lib/logstash/outputs/graphite.rb, line 148
def messages_from_event_fields(event, include_metrics, exclude_metrics)
  @logger.debug? && @logger.debug("got metrics event", :metrics => event.to_hash)

  timestamp = event_timestamp(event)
  event.to_hash.flat_map do |metric,value|
    next if EXCLUDE_ALWAYS.include?(metric)
    next unless include_metrics.empty? || include_metrics.any? { |regexp| metric.match(regexp) }
    next if exclude_metrics.any? {|regexp| metric.match(regexp)}

    metrics_lines_for_event(event, metric, value, timestamp)
  end
end
messages_from_event_metrics(event, metrics) click to toggle source
# File lib/logstash/outputs/graphite.rb, line 161
def messages_from_event_metrics(event, metrics)
  timestamp = event_timestamp(event)
  metrics.flat_map do |metric, value|
    @logger.debug? && @logger.debug("processing", :metric => metric, :value => value)

    metric = event.sprintf(metric)
    next unless @include_metrics.any? {|regexp| metric.match(regexp)}
    next if @exclude_metrics.any? {|regexp| metric.match(regexp)}

    metrics_lines_for_event(event, metric, value, timestamp)
  end
end
metrics_line(event, name, value, timestamp) click to toggle source
# File lib/logstash/outputs/graphite.rb, line 188
def metrics_line(event, name, value, timestamp)
  "#{construct_metric_name(event, name)} #{value} #{timestamp}"
end
metrics_lines_for_event(event, metric, value, timestamp) click to toggle source
# File lib/logstash/outputs/graphite.rb, line 178
def metrics_lines_for_event(event, metric, value, timestamp)
  if event.get(metric).is_a?(Hash)
    dotify(event.get(metric), metric).map do |k, v|
      metrics_line(event, k, v, timestamp)
    end
  else
    metrics_line(event, event.sprintf(metric), event.sprintf(value).to_f, timestamp)
  end
end