class NewRelic::Agent::LogEventAggregator

Constants

CUSTOM_ATTRIBUTES_KEY
DECORATING_ENABLED_KEY
DECORATING_SUPPORTABILITY_FORMAT
DROPPED_METRIC
FORWARDING_ENABLED_KEY
FORWARDING_SUPPORTABILITY_FORMAT
LEVEL_KEY

Per-message keys

LINES

Metric keys

LOG_LEVEL_KEY
MAX_BYTES
MESSAGE_KEY
METRICS_ENABLED_KEY
METRICS_SUPPORTABILITY_FORMAT
OVERALL_ENABLED_KEY

Config keys

OVERALL_SUPPORTABILITY_FORMAT
PRIORITY_KEY
SEEN_METRIC
SENT_METRIC
TIMESTAMP_KEY

Attributes

attributes[R]

Public Class Methods

new(events) click to toggle source
Calls superclass method NewRelic::Agent::EventAggregator::new
# File lib/new_relic/agent/log_event_aggregator.rb, line 45
def initialize(events)
  super(events)
  @counter_lock = Mutex.new
  @seen = 0
  @seen_by_severity = Hash.new(0)
  @high_security = NewRelic::Agent.config[:high_security]
  @instrumentation_logger_enabled = NewRelic::Agent::Instrumentation::Logger.enabled?
  @attributes = NewRelic::Agent::LogEventAttributes.new
  register_for_done_configuring(events)
end
payload_to_melt_format(data) click to toggle source

Because our transmission format (MELT) is different than historical agent payloads, extract the munging here to keep the service focused on the general harvest + transmit instead of the format.

Payload shape matches the publicly documented MELT format. docs.newrelic.com/docs/logs/log-api/introduction-log-api

We have to keep the aggregated payloads in a separate shape, though, to work with the priority sampling buffers

# File lib/new_relic/agent/log_event_aggregator.rb, line 137
def self.payload_to_melt_format(data)
  common_attributes = LinkingMetadata.append_service_linking_metadata({})

  # To save on unnecessary data transmission, trim the entity.type
  # sent by classic logs-in-context
  common_attributes.delete(ENTITY_TYPE_KEY)

  common_attributes.merge!(NewRelic::Agent.agent.log_event_aggregator.attributes.custom_attributes)

  _, items = data
  payload = [{
    common: {attributes: common_attributes},
    logs: items.map(&:last)
  }]

  return [payload, items.size]
end

Public Instance Methods

add_custom_attributes(custom_attributes) click to toggle source
# File lib/new_relic/agent/log_event_aggregator.rb, line 124
def add_custom_attributes(custom_attributes)
  attributes.add_custom_attributes(custom_attributes)
end
capacity() click to toggle source
# File lib/new_relic/agent/log_event_aggregator.rb, line 56
def capacity
  @buffer.capacity
end
create_event(priority, formatted_message, severity) click to toggle source
# File lib/new_relic/agent/log_event_aggregator.rb, line 107
def create_event(priority, formatted_message, severity)
  formatted_message = truncate_message(formatted_message)

  event = LinkingMetadata.append_trace_linking_metadata({
    LEVEL_KEY => severity,
    MESSAGE_KEY => formatted_message,
    TIMESTAMP_KEY => Process.clock_gettime(Process::CLOCK_REALTIME) * 1000
  })

  [
    {
      PrioritySampledBuffer::PRIORITY_KEY => priority
    },
    event
  ]
end
enabled?() click to toggle source
# File lib/new_relic/agent/log_event_aggregator.rb, line 169
def enabled?
  @enabled && @instrumentation_logger_enabled
end
harvest!() click to toggle source
# File lib/new_relic/agent/log_event_aggregator.rb, line 155
def harvest!
  record_customer_metrics()
  super
end
record(formatted_message, severity) click to toggle source
# File lib/new_relic/agent/log_event_aggregator.rb, line 60
def record(formatted_message, severity)
  return unless enabled?

  severity = 'UNKNOWN' if severity.nil? || severity.empty?

  if NewRelic::Agent.config[METRICS_ENABLED_KEY]
    @counter_lock.synchronize do
      @seen += 1
      @seen_by_severity[severity] += 1
    end
  end

  return if severity_too_low?(severity)
  return if formatted_message.nil? || formatted_message.empty?
  return unless NewRelic::Agent.config[FORWARDING_ENABLED_KEY]
  return if @high_security

  txn = NewRelic::Agent::Transaction.tl_current
  priority = LogPriority.priority_for(txn)

  if txn
    return txn.add_log_event(create_event(priority, formatted_message, severity))
  else
    return @lock.synchronize do
      @buffer.append(priority: priority) do
        create_event(priority, formatted_message, severity)
      end
    end
  end
rescue
  nil
end
record_batch(txn, logs) click to toggle source
# File lib/new_relic/agent/log_event_aggregator.rb, line 93
def record_batch(txn, logs)
  # Ensure we have the same shared priority
  priority = LogPriority.priority_for(txn)
  logs.each do |log|
    log.first[PRIORITY_KEY] = priority
  end

  @lock.synchronize do
    logs.each do |log|
      @buffer.append(event: log)
    end
  end
end
reset!() click to toggle source
Calls superclass method NewRelic::Agent::EventAggregator#reset!
# File lib/new_relic/agent/log_event_aggregator.rb, line 160
def reset!
  @counter_lock.synchronize do
    @seen = 0
    @seen_by_severity.clear
  end

  super
end

Private Instance Methods

after_harvest(metadata) click to toggle source
# File lib/new_relic/agent/log_event_aggregator.rb, line 200
def after_harvest(metadata)
  dropped_count = metadata[:seen] - metadata[:captured]
  note_dropped_events(metadata[:seen], dropped_count)
  record_supportability_metrics(metadata[:seen], metadata[:captured], dropped_count)
end
configured_log_level_constant() click to toggle source
# File lib/new_relic/agent/log_event_aggregator.rb, line 250
def configured_log_level_constant
  format_log_level_constant(NewRelic::Agent.config[LOG_LEVEL_KEY])
end
format_log_level_constant(log_level) click to toggle source
# File lib/new_relic/agent/log_event_aggregator.rb, line 254
def format_log_level_constant(log_level)
  log_level.upcase.to_sym
end
line_metric_name_by_severity(severity) click to toggle source
# File lib/new_relic/agent/log_event_aggregator.rb, line 225
def line_metric_name_by_severity(severity)
  @line_metrics ||= {}
  @line_metrics[severity] ||= "Logging/lines/#{severity}".freeze
end
note_dropped_events(total_count, dropped_count) click to toggle source
# File lib/new_relic/agent/log_event_aggregator.rb, line 230
def note_dropped_events(total_count, dropped_count)
  if dropped_count > 0
    NewRelic::Agent.logger.warn("Dropped #{dropped_count} log events out of #{total_count}.")
  end
end
record_configuration_metric(format, key) click to toggle source
# File lib/new_relic/agent/log_event_aggregator.rb, line 190
def record_configuration_metric(format, key)
  state = NewRelic::Agent.config[key]
  label = if !enabled?
    'disabled'
  else
    state ? 'enabled' : 'disabled'
  end
  NewRelic::Agent.increment_metric(format % label)
end
record_customer_metrics() click to toggle source

To avoid paying the cost of metric recording on every line, we hold these until harvest before recording them

# File lib/new_relic/agent/log_event_aggregator.rb, line 208
def record_customer_metrics
  return unless enabled?
  return unless NewRelic::Agent.config[METRICS_ENABLED_KEY]

  @counter_lock.synchronize do
    return unless @seen > 0

    NewRelic::Agent.increment_metric(LINES, @seen)
    @seen_by_severity.each do |(severity, count)|
      NewRelic::Agent.increment_metric(line_metric_name_by_severity(severity), count)
    end

    @seen = 0
    @seen_by_severity.clear
  end
end
record_supportability_metrics(total_count, captured_count, dropped_count) click to toggle source
# File lib/new_relic/agent/log_event_aggregator.rb, line 236
def record_supportability_metrics(total_count, captured_count, dropped_count)
  return unless total_count > 0

  NewRelic::Agent.increment_metric(DROPPED_METRIC, dropped_count)
  NewRelic::Agent.increment_metric(SEEN_METRIC, total_count)
  NewRelic::Agent.increment_metric(SENT_METRIC, captured_count)
end
register_for_done_configuring(events) click to toggle source

We record once-per-connect metrics for enabled/disabled state at the point we consider the configuration stable (i.e. once we’ve gotten SSC)

# File lib/new_relic/agent/log_event_aggregator.rb, line 177
def register_for_done_configuring(events)
  events.subscribe(:server_source_configuration_added) do
    @high_security = NewRelic::Agent.config[:high_security]

    record_configuration_metric(OVERALL_SUPPORTABILITY_FORMAT, OVERALL_ENABLED_KEY)
    record_configuration_metric(METRICS_SUPPORTABILITY_FORMAT, METRICS_ENABLED_KEY)
    record_configuration_metric(FORWARDING_SUPPORTABILITY_FORMAT, FORWARDING_ENABLED_KEY)
    record_configuration_metric(DECORATING_SUPPORTABILITY_FORMAT, DECORATING_ENABLED_KEY)

    add_custom_attributes(NewRelic::Agent.config[CUSTOM_ATTRIBUTES_KEY])
  end
end
severity_too_low?(severity) click to toggle source
# File lib/new_relic/agent/log_event_aggregator.rb, line 258
def severity_too_low?(severity)
  severity_constant = format_log_level_constant(severity)
  # always record custom log levels
  return false unless Logger::Severity.constants.include?(severity_constant)

  Logger::Severity.const_get(severity_constant) < Logger::Severity.const_get(configured_log_level_constant)
end
truncate_message(message) click to toggle source
# File lib/new_relic/agent/log_event_aggregator.rb, line 244
def truncate_message(message)
  return message if message.bytesize <= MAX_BYTES

  message.byteslice(0...MAX_BYTES)
end