class LogStash::Outputs::Loggly
Got a loggly account? Use logstash to ship logs to Loggly
!
This is most useful so you can use logstash to parse and structure your logs and ship structured, json events to your Loggly
account.
To use this, you'll need to use a Loggly
input with type 'http' and 'json logging' enabled.
Constants
- HTTP_FORBIDDEN
- HTTP_GATEWAY_TIMEOUT
- HTTP_INTERNAL_SERVER_ERROR
- HTTP_NOT_FOUND
- HTTP_SUCCESS
HTTP constants
Public Instance Methods
Concatenates JSON events to build an API call body.
Will yield before going over the body size limit. May yield more than once.
This is also where we check that each message respects the message size, and where we skip those if they don't.
# File lib/logstash/outputs/loggly.rb, line 214 def build_message_bodies(events) body = '' event_count = 0 events.each do |event| encoded_event = format_message(event) event_size = encoded_event.bytesize if event_size > @max_event_size @logger.warn "Skipping event over max event size", :event_size => encoded_event.bytesize, :max_event_size => @max_event_size @logger.debug "Skipped event", :event => encoded_event next end if body.bytesize + 1 + event_size > @max_payload_size @logger.debug "Flushing events to Loggly", count: event_count, bytes: body.bytesize yield body body = '' event_count = 0 end body << "\n" unless body.bytesize.zero? body << encoded_event event_count += 1 end if event_count > 0 @logger.debug "Flushing events to Loggly", count: event_count, bytes: body.bytesize yield body end end
# File lib/logstash/outputs/loggly.rb, line 169 def format_message(event) event.to_json end
# File lib/logstash/outputs/loggly.rb, line 126 def multi_receive(events) send_batch events.collect { |event| prepare_meta(event) } end
# File lib/logstash/outputs/loggly.rb, line 130 def receive(event) send_batch [prepare_meta(event)] end
# File lib/logstash/outputs/loggly.rb, line 121 def register @logger.debug "Initializing Loggly Output", @config end
Takes an array of meta_events or nils. Will split the batch in appropriate sub-batches per key+tag combination (which need to be posted to different URIs).
# File lib/logstash/outputs/loggly.rb, line 175 def send_batch(meta_events) split_batches(meta_events.compact).each_pair do |k, batch| key, tag = *k if tag.nil? url = "#{@proto}://#{@host}/bulk/#{key}" else url = "#{@proto}://#{@host}/bulk/#{key}/tag/#{tag}" end build_message_bodies(batch) do |body| perform_api_call url, body end end end
Gets all API calls to the same URI together in common batches.
Expects an array of meta_events {key: '…', tag: '…', event: event } Outputs a hash with event batches split out by key+tag combination.
{ [key1, tag1] => [event1, ...], [key2, tag1] => [...], [key2, tag2] => [...], ... }
# File lib/logstash/outputs/loggly.rb, line 199 def split_batches(events) events.reduce( Hash.new { |h,k| h[k] = [] } ) do |acc, meta_event| key = meta_event[:key] tag = meta_event[:tag] acc[ [key, tag] ] << meta_event[:event] acc end end
Private Instance Methods
# File lib/logstash/outputs/loggly.rb, line 248 def perform_api_call(url, message) url = URI.parse(url) http = Net::HTTP::Proxy(@proxy_host, @proxy_port, @proxy_user, @proxy_password.value).new(url.host, url.port) if url.scheme == 'https' http.use_ssl = true http.verify_mode = OpenSSL::SSL::VERIFY_NONE end request = Net::HTTP::Post.new(url.path, {'Content-Type' => @mime_type}) request.body = message # Variable for count total retries totalRetries = 0 #try posting once when can_retry is false if @can_retry == false @retry_count = 1 end @retry_count.times do begin response = http.request(request) @logger.debug("Loggly response", code: response.code, body: response.body) case response.code # HTTP_SUCCESS :Code 2xx when HTTP_SUCCESS @logger.debug("Event batch sent successfully") # HTTP_FORBIDDEN :Code 403 when HTTP_FORBIDDEN @logger.warn("User does not have privileges to execute the action.") # HTTP_NOT_FOUND :Code 404 when HTTP_NOT_FOUND @logger.warn("Invalid URL. Please check URL should be http://logs-01.loggly.com/inputs/CUSTOMER_TOKEN/tag/TAG", :url => url.to_s) # HTTP_INTERNAL_SERVER_ERROR :Code 500 when HTTP_INTERNAL_SERVER_ERROR @logger.warn("Internal Server Error") # HTTP_GATEWAY_TIMEOUT :Code 504 when HTTP_GATEWAY_TIMEOUT @logger.warn("Gateway Time Out") else @logger.error("Unexpected response code", :code => response.code) end # case if [HTTP_SUCCESS,HTTP_FORBIDDEN,HTTP_NOT_FOUND].include?(response.code) # break the retries loop for the specified response code break end rescue StandardError => e @logger.error("An unexpected error occurred", :exception => e.class.name, :error => e.to_s, :backtrace => e.backtrace) end # rescue if totalRetries < @retry_count && totalRetries > 0 @logger.warn "Waiting for five seconds before retry..." sleep(5) end totalRetries = totalRetries + 1 end #loop end
Returns one meta event {key: '…', tag: '…', event: event }, or returns nil, if event's key doesn't resolve.
# File lib/logstash/outputs/loggly.rb, line 137 def prepare_meta(event) key = event.sprintf(@key) tags = @tag.split(",") tag_array = [] tags.each do |t| t = event.sprintf(t) # For those cases where %{somefield} doesn't exist we don't include it unless /%{\w+}/.match(t) || t.blank? tag_array.push(t) end end if expected_field = key[/%{(.*)}/, 1] @logger.warn "Skipping sending message to Loggly. No key provided (key='#{key}'). Make sure to set field '#{expected_field}'." @logger.debug "Dropped message", :event => event.to_json return nil end unless tag_array.empty? tag = tag_array.uniq.join(",") end event_hash = event.to_hash # Don't want to modify the event in an output if @convert_timestamp && event_hash['@timestamp'] && !event_hash['timestamp'] event_hash['timestamp'] = event_hash.delete('@timestamp') end meta_event = { key: key, tag: tag, event: event_hash } end