module Spidy::Definition
Class representing a website defined by DSL
Public Instance Methods
call(source = nil, name: :default, &yielder)
click to toggle source
# File lib/spidy/definition.rb, line 11 def call(source = nil, name: :default, &yielder) name = name.presence || :default spidy = @namespace[:"#{name}_scraper"] fail "undefined spidy [#{name}]" if spidy.nil? spidy.call(source, &yielder) end
define(name = :default, connector: nil, binder: nil, as: nil, &define_block)
click to toggle source
# File lib/spidy/definition.rb, line 47 def define(name = :default, connector: nil, binder: nil, as: nil, &define_block) @namespace ||= {} connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent, socks_proxy: @socks_proxy) binder = Spidy::Binder.get(self, binder || as) @namespace[:"#{name}_scraper"] = define_proc(name, connector, binder, define_block) end
each(source = nil, name: :default, &yielder)
click to toggle source
# File lib/spidy/definition.rb, line 31 def each(source = nil, name: :default, &yielder) name = name.presence || :default spidy = @namespace[:"#{name}_spider"] fail "undefined spidy [#{name}]" if spidy.nil? spidy.call(source, &yielder) end
namespace()
click to toggle source
# File lib/spidy/definition.rb, line 7 def namespace @namespace end
socks_proxy(host, port)
click to toggle source
# File lib/spidy/definition.rb, line 27 def socks_proxy(host, port) @socks_proxy = { host: host, port: port } end
spider(name = :default, connector: nil, as: nil, &define_block)
click to toggle source
# File lib/spidy/definition.rb, line 39 def spider(name = :default, connector: nil, as: nil, &define_block) @namespace ||= {} connector = Spidy::Connector.get(connector || as, wait_time: @wait_time, user_agent: @user_agent, socks_proxy: @socks_proxy) @namespace[:"#{name}_spider"] = proc do |source, &yielder| define_block.call(yielder, connector, source) end end
user_agent(user_agent)
click to toggle source
# File lib/spidy/definition.rb, line 19 def user_agent(user_agent) @user_agent = user_agent end
wait_time(wait_time)
click to toggle source
# File lib/spidy/definition.rb, line 23 def wait_time(wait_time) @wait_time = wait_time end
Private Instance Methods
define_proc(name, connector, binder, define_block)
click to toggle source
# File lib/spidy/definition.rb, line 56 def define_proc(name, connector, binder, define_block) proc do |source, &yielder| yielder = lambda { |result| break result } if yielder.nil? connection_yielder = lambda do |page| binder.call(page, url: source, define: define_block, define_name: name) { |object| yielder.call(object) } end connector.call(source, &connection_yielder) end end