module Spidy::Definition

Class representing a website defined by DSL

Public Instance Methods

call(source = nil, name: :default, &yielder) click to toggle source
# File lib/spidy/definition.rb, line 11
def call(source = nil, name: :default, &yielder)
  name = name.presence || :default
  spidy = @namespace[:"#{name}_scraper"]
  fail "undefined spidy [#{name}]" if spidy.nil?

  spidy.call(source, &yielder)
end
define(name = :default, connector: nil, binder: nil, as: nil, &define_block) click to toggle source
# File lib/spidy/definition.rb, line 47
def define(name = :default, connector: nil, binder: nil, as: nil, &define_block)
  @namespace ||= {}
  connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent, socks_proxy: @socks_proxy)
  binder = Spidy::Binder.get(self, binder || as)
  @namespace[:"#{name}_scraper"] = define_proc(name, connector, binder, define_block)
end
each(source = nil, name: :default, &yielder) click to toggle source
# File lib/spidy/definition.rb, line 31
def each(source = nil, name: :default, &yielder)
  name = name.presence || :default
  spidy = @namespace[:"#{name}_spider"]
  fail "undefined spidy [#{name}]" if spidy.nil?

  spidy.call(source, &yielder)
end
namespace() click to toggle source
# File lib/spidy/definition.rb, line 7
def namespace
  @namespace
end
socks_proxy(host, port) click to toggle source
# File lib/spidy/definition.rb, line 27
def socks_proxy(host, port)
  @socks_proxy = { host: host, port: port }
end
spider(name = :default, connector: nil, as: nil, &define_block) click to toggle source
# File lib/spidy/definition.rb, line 39
def spider(name = :default, connector: nil, as: nil, &define_block)
  @namespace ||= {}
  connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent, socks_proxy: @socks_proxy)
  @namespace[:"#{name}_spider"] = proc do |source, &yielder|
    define_block.call(yielder, connector, source)
  end
end
user_agent(user_agent) click to toggle source
# File lib/spidy/definition.rb, line 19
def user_agent(user_agent)
  @user_agent = user_agent
end
wait_time(wait_time) click to toggle source
# File lib/spidy/definition.rb, line 23
def wait_time(wait_time)
  @wait_time = wait_time
end

Private Instance Methods

define_proc(name, connector, binder, define_block) click to toggle source
# File lib/spidy/definition.rb, line 56
def define_proc(name, connector, binder, define_block)
  proc do |source, &yielder|
    yielder = lambda { |result| break result } if yielder.nil?
    connection_yielder = lambda do |page|
      binder.call(page, url: source, define: define_block, define_name: name) { |object| yielder.call(object) }
    end
    connector.call(source, &connection_yielder)
  end
end