class Anemone::Tentacle

Public Class Methods

new(link_queue, page_queue, opts = {}) click to toggle source

Create a new Tentacle

# File lib/anemone/tentacle.rb, line 9
def initialize(link_queue, page_queue, opts = {})
  @link_queue = link_queue
  @page_queue = page_queue
  @http = Anemone::HTTP.new(opts)
  @opts = opts
end

Public Instance Methods

run() click to toggle source

Gets links from @link_queue, and returns the fetched Page objects into @page_queue

# File lib/anemone/tentacle.rb, line 20
def run
  loop do
    link, referer, depth = @link_queue.deq

    break if link == :END

    @http.fetch_pages(link, referer, depth).each { |page| @page_queue << page }

    delay
  end
end

Private Instance Methods

delay() click to toggle source
# File lib/anemone/tentacle.rb, line 34
def delay
  sleep @opts[:delay] if @opts[:delay] > 0
end