class Kudzu::Agent::Sleeper

Public Class Methods

new(config, robots = nil) click to toggle source
# File lib/kudzu/agent/sleeper.rb, line 4
def initialize(config, robots = nil)
  @config = config
  @robots = robots
  @monitor = Monitor.new
  @last_accessed = {}
end

Public Instance Methods

politeness_delay(uri) click to toggle source
# File lib/kudzu/agent/sleeper.rb, line 11
def politeness_delay(uri)
  uri = Addressable::URI.parse(uri) if uri.is_a?(String)
  delay_sec = delay_second(uri)
  return unless delay_sec

  sleep_sec = sleep_second(uri, delay_sec)
  sleep sleep_sec if sleep_sec > 0
end

Private Instance Methods

delay_second(uri) click to toggle source
# File lib/kudzu/agent/sleeper.rb, line 22
def delay_second(uri)
  if @config.respect_robots_txt && @robots && (crawl_delay = @robots.crawl_delay(uri))
    crawl_delay.to_f
  elsif @config.politeness_delay
    @config.politeness_delay.to_f
  end
end
sleep_second(uri, delay_sec) click to toggle source
# File lib/kudzu/agent/sleeper.rb, line 30
def sleep_second(uri, delay_sec)
  @monitor.synchronize do
    now = Time.now.to_f
    value = if @last_accessed[uri.host]
              (@last_accessed[uri.host] + delay_sec) - now
            else
              0
            end
    @last_accessed[uri.host] = now + value
    value
  end
end