class LogStash::Outputs::Loggly

Got a loggly account? Use logstash to ship logs to Loggly!

This is most useful so you can use logstash to parse and structure your logs and ship structured, json events to your Loggly account.

To use this, you'll need to use a Loggly input with type 'http' and 'json logging' enabled.

Constants

HTTP_FORBIDDEN
HTTP_GATEWAY_TIMEOUT
HTTP_INTERNAL_SERVER_ERROR
HTTP_NOT_FOUND
HTTP_SUCCESS

HTTP constants

Public Instance Methods

build_message_bodies(events) { |body| ... } click to toggle source

Concatenates JSON events to build an API call body.

Will yield before going over the body size limit. May yield more than once.

This is also where we check that each message respects the message size, and where we skip those if they don't.

# File lib/logstash/outputs/loggly.rb, line 214
def build_message_bodies(events)
  body = ''
  event_count = 0

  events.each do |event|
    encoded_event = format_message(event)
    event_size = encoded_event.bytesize

    if event_size > @max_event_size
      @logger.warn "Skipping event over max event size",
        :event_size => encoded_event.bytesize, :max_event_size => @max_event_size
      @logger.debug "Skipped event", :event => encoded_event
      next
    end

    if body.bytesize + 1 + event_size > @max_payload_size
      @logger.debug "Flushing events to Loggly", count: event_count, bytes: body.bytesize
      yield body
      body = ''
      event_count = 0
    end

    body << "\n" unless body.bytesize.zero?
    body << encoded_event
    event_count += 1
  end

  if event_count > 0
    @logger.debug "Flushing events to Loggly", count: event_count, bytes: body.bytesize
    yield body
  end
end
format_message(event) click to toggle source
# File lib/logstash/outputs/loggly.rb, line 169
def format_message(event)
  event.to_json
end
multi_receive(events) click to toggle source
# File lib/logstash/outputs/loggly.rb, line 126
def multi_receive(events)
  send_batch events.collect { |event| prepare_meta(event) }
end
receive(event) click to toggle source
# File lib/logstash/outputs/loggly.rb, line 130
def receive(event)
  send_batch [prepare_meta(event)]
end
register() click to toggle source
# File lib/logstash/outputs/loggly.rb, line 121
def register
  @logger.debug "Initializing Loggly Output", @config
end
send_batch(meta_events) click to toggle source

Takes an array of meta_events or nils. Will split the batch in appropriate sub-batches per key+tag combination (which need to be posted to different URIs).

# File lib/logstash/outputs/loggly.rb, line 175
def send_batch(meta_events)
  split_batches(meta_events.compact).each_pair do |k, batch|
    key, tag = *k
    if tag.nil?
      url = "#{@proto}://#{@host}/bulk/#{key}"
    else
      url = "#{@proto}://#{@host}/bulk/#{key}/tag/#{tag}"
    end


    build_message_bodies(batch) do |body|
      perform_api_call url, body
    end
  end
end
split_batches(events) click to toggle source

Gets all API calls to the same URI together in common batches.

Expects an array of meta_events {key: '…', tag: '…', event: event } Outputs a hash with event batches split out by key+tag combination.

{ [key1, tag1] => [event1, ...],
  [key2, tag1] => [...],
  [key2, tag2] => [...],
  ... }
# File lib/logstash/outputs/loggly.rb, line 199
def split_batches(events)
  events.reduce( Hash.new { |h,k| h[k] = [] } ) do |acc, meta_event|
    key = meta_event[:key]
    tag = meta_event[:tag]
    acc[ [key, tag] ] << meta_event[:event]
    acc
  end
end

Private Instance Methods

perform_api_call(url, message) click to toggle source
# File lib/logstash/outputs/loggly.rb, line 248
def perform_api_call(url, message)
  url = URI.parse(url)

  http = Net::HTTP::Proxy(@proxy_host,
                          @proxy_port,
                          @proxy_user,
                          @proxy_password.value).new(url.host, url.port)

  if url.scheme == 'https'
    http.use_ssl = true
    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
  end

  request = Net::HTTP::Post.new(url.path, {'Content-Type' => @mime_type})
  request.body = message

  # Variable for count total retries
  totalRetries = 0

  #try posting once when can_retry is false
  if @can_retry == false
    @retry_count = 1
  end


  @retry_count.times do
    begin
      response = http.request(request)
      @logger.debug("Loggly response", code: response.code, body: response.body)

      case response.code

      # HTTP_SUCCESS :Code 2xx
      when HTTP_SUCCESS
        @logger.debug("Event batch sent successfully")

      # HTTP_FORBIDDEN :Code 403
      when HTTP_FORBIDDEN
        @logger.warn("User does not have privileges to execute the action.")

      # HTTP_NOT_FOUND :Code 404
      when HTTP_NOT_FOUND
        @logger.warn("Invalid URL. Please check URL should be http://logs-01.loggly.com/inputs/CUSTOMER_TOKEN/tag/TAG", :url => url.to_s)

      # HTTP_INTERNAL_SERVER_ERROR :Code 500
      when HTTP_INTERNAL_SERVER_ERROR
        @logger.warn("Internal Server Error")

      # HTTP_GATEWAY_TIMEOUT :Code 504
      when HTTP_GATEWAY_TIMEOUT
        @logger.warn("Gateway Time Out")
      else
        @logger.error("Unexpected response code", :code => response.code)
      end # case

      if [HTTP_SUCCESS,HTTP_FORBIDDEN,HTTP_NOT_FOUND].include?(response.code)   # break the retries loop for the specified response code
        break
      end

    rescue StandardError => e
      @logger.error("An unexpected error occurred", :exception => e.class.name, :error => e.to_s, :backtrace => e.backtrace)
    end # rescue

    if totalRetries < @retry_count && totalRetries > 0
      @logger.warn "Waiting for five seconds before retry..."
      sleep(5)
    end

    totalRetries = totalRetries + 1
  end #loop
end
prepare_meta(event) click to toggle source

Returns one meta event {key: '…', tag: '…', event: event }, or returns nil, if event's key doesn't resolve.

# File lib/logstash/outputs/loggly.rb, line 137
def prepare_meta(event)
  key = event.sprintf(@key)
  tags = @tag.split(",")
  tag_array = []

  tags.each do |t|
    t = event.sprintf(t)
    # For those cases where %{somefield} doesn't exist we don't include it
    unless /%{\w+}/.match(t) || t.blank?
      tag_array.push(t)
    end
  end

  if expected_field = key[/%{(.*)}/, 1]
    @logger.warn "Skipping sending message to Loggly. No key provided (key='#{key}'). Make sure to set field '#{expected_field}'."
    @logger.debug "Dropped message", :event => event.to_json
    return nil
  end

  unless tag_array.empty?
    tag = tag_array.uniq.join(",")
  end

  event_hash = event.to_hash # Don't want to modify the event in an output
  if @convert_timestamp && event_hash['@timestamp'] && !event_hash['timestamp']
    event_hash['timestamp'] = event_hash.delete('@timestamp')
  end

  meta_event = {  key: key, tag: tag, event: event_hash }
end