class Mmtrix::Agent::Threading::BacktraceService
Constants
- ALL_TRANSACTIONS
- MAX_BUFFER_LENGTH
Attributes
This method is expected to be called with @lock held.
Public Class Methods
Because of Resque’s forking, we don’t poll thread backtraces for it. To accomplish that would require starting a new backtracing thread in each forked worker, and merging profiles across the pipe channel.
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 18 def self.is_resque? Mmtrix::Agent.config[:dispatcher] == :resque end
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 11 def self.is_supported? RUBY_VERSION >= "1.9.2" && !is_resque? end
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 26 def initialize(event_listener=nil) @profiles = {} @buffer = {} # synchronizes access to @profiles and @buffer above @lock = Mutex.new @running = false @profile_agent_code = false @worker_loop = Mmtrix::Agent::WorkerLoop.new # Memoize overhead % to avoid getting stale OR looked up every poll @overhead_percent_threshold = Mmtrix::Agent.config[:'xray_session.max_profile_overhead'] Mmtrix::Agent.config.register_callback(:'xray_session.max_profile_overhead') do |new_value| @overhead_percent_threshold = new_value end if event_listener event_listener.subscribe(:transaction_finished, &method(:on_transaction_finished)) end end
Public Instance Methods
If our overhead % exceeds the threshold, bump the next poll period relative to how much larger our overhead is than allowed
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 251 def adjust_polling_time(now, poll_start) duration = now - poll_start overhead_percent = duration / effective_polling_period if overhead_percent > self.overhead_percent_threshold scale_up_by = overhead_percent / self.overhead_percent_threshold worker_loop.period = effective_polling_period * scale_up_by else worker_loop.period = effective_polling_period end end
This method is expected to be called with @lock held.
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 131 def aggregate_backtraces(backtraces, name, start, duration, bucket, thread) end_time = start + duration backtraces.each do |(timestamp, backtrace)| if timestamp >= start && timestamp < end_time @profiles[name].aggregate(backtrace, bucket, thread) end end end
This method is expected to be called with @lock held.
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 221 def aggregate_global_backtrace(backtrace, bucket, thread) if @profiles[ALL_TRANSACTIONS] @profiles[ALL_TRANSACTIONS].aggregate(backtrace, bucket, thread) end end
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 202 def allowed_bucket?(bucket) bucket == :request || bucket == :background end
This method is expected to be called with @lock held.
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 209 def buffer_backtrace_for_thread(thread, timestamp, backtrace, bucket) if should_buffer?(bucket) @buffer[thread] ||= [] if @buffer[thread].length < MAX_BUFFER_LENGTH @buffer[thread] << [timestamp, backtrace] else Mmtrix::Agent.increment_metric('Supportability/XraySessions/DroppedBacktraces') end end end
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 159 def effective_polling_period=(new_period) @effective_polling_period = new_period self.worker_loop.period = new_period end
This method is expected to be called with @lock held.
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 240 def find_effective_polling_period @profiles.values.map { |p| p.requested_period }.min end
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 103 def harvest(transaction_name) @lock.synchronize do if @profiles[transaction_name] profile = @profiles.delete(transaction_name) profile.finished_at = Time.now @profiles[transaction_name] = ThreadProfile.new(profile.command_arguments) profile end end end
This method is expected to be called with @lock held.
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 189 def need_backtrace?(bucket) ( bucket != :ignore && (@profiles[ALL_TRANSACTIONS] || should_buffer?(bucket)) ) end
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 114 def on_transaction_finished(payload) name = payload[:name] start = payload[:start_timestamp] duration = payload[:duration] thread = payload[:thread] || Thread.current bucket = payload[:bucket] @lock.synchronize do backtraces = @buffer.delete(thread) if backtraces && @profiles.has_key?(name) aggregate_backtraces(backtraces, name, start, duration, bucket, thread) end end end
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 164 def poll poll_start = Time.now @lock.synchronize do AgentThread.list.each do |thread| sample_thread(thread) end @profiles.each_value { |p| p.increment_poll_count } @buffer.delete_if { |thread, _| !thread.alive? } end end_time = Time.now adjust_polling_time(end_time, poll_start) record_supportability_metrics(end_time, poll_start) end
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 268 def record_polling_time(now, poll_start) Mmtrix::Agent.record_metric('Supportability/ThreadProfiler/PollingTime', now - poll_start) end
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 272 def record_skew(poll_start) if @last_poll skew = poll_start - @last_poll - worker_loop.period Mmtrix::Agent.record_metric('Supportability/ThreadProfiler/Skew', skew) end @last_poll = poll_start end
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 263 def record_supportability_metrics(now, poll_start) record_polling_time(now, poll_start) record_skew(poll_start) end
Public interface
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 50 def running? @running end
This method is expected to be called with @lock held.
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 228 def sample_thread(thread) bucket = AgentThread.bucket_thread(thread, @profile_agent_code) if need_backtrace?(bucket) timestamp = Time.now.to_f backtrace = AgentThread.scrub_backtrace(thread, @profile_agent_code) aggregate_global_backtrace(backtrace, bucket, thread) buffer_backtrace_for_thread(thread, timestamp, backtrace, bucket) end end
This method is expected to be called with @lock held.
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 184 def should_buffer?(bucket) allowed_bucket?(bucket) && watching_for_transaction? end
This method is expected to be called with @lock held.
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 245 def should_profile_agent_code? @profiles.values.any? { |p| p.profile_agent_code } end
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 140 def start return if @running || !self.class.is_supported? @running = true self.worker_thread = AgentThread.create('Backtrace Service') do # Not passing period because we expect it's already been set. self.worker_loop.run(&method(:poll)) end end
This method is expected to be called with @lock held
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 151 def stop return unless @running @running = false self.worker_loop.stop @buffer = {} end
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 54 def subscribe(transaction_name, command_arguments={}) if self.class.is_resque? Mmtrix::Agent.logger.info("Backtracing threads on Resque is not supported, so not subscribing transaction '#{transaction_name}'") return end if !self.class.is_supported? Mmtrix::Agent.logger.debug("Backtracing not supported, so not subscribing transaction '#{transaction_name}'") return end Mmtrix::Agent.logger.debug("Backtrace Service subscribing transaction '#{transaction_name}'") profile = ThreadProfile.new(command_arguments) @lock.synchronize do @profiles[transaction_name] = profile update_values_from_profiles end start profile end
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 97 def subscribed?(transaction_name) @lock.synchronize do @profiles.has_key?(transaction_name) end end
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 78 def unsubscribe(transaction_name) return unless self.class.is_supported? Mmtrix::Agent.logger.debug("Backtrace Service unsubscribing transaction '#{transaction_name}'") @lock.synchronize do @profiles.delete(transaction_name) if @profiles.empty? stop else update_values_from_profiles end end end
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 92 def update_values_from_profiles self.effective_polling_period = find_effective_polling_period self.profile_agent_code = should_profile_agent_code? end
This method is expected to be called with @lock held
# File lib/mmtrix/agent/threading/backtrace_service.rb, line 197 def watching_for_transaction? @profiles.size > 1 || (@profiles.size == 1 && @profiles[ALL_TRANSACTIONS].nil?) end