module Spidr

Constants

VERSION

Spidr version

Public Class Methods

domain(name,**kwargs,&block) click to toggle source

@see Agent.domain

@since 0.7.0

# File lib/spidr/spidr.rb, line 57
def self.domain(name,**kwargs,&block)
  Agent.domain(name,**kwargs,&block)
end
host(name,**kwargs,&block) click to toggle source

@see Agent.host

# File lib/spidr/spidr.rb, line 48
def self.host(name,**kwargs,&block)
  Agent.host(name,**kwargs,&block)
end
robots() click to toggle source

@abstract

# File lib/spidr/spidr.rb, line 71
def self.robots
end
robots=(mode) click to toggle source

Enables or disables `robots.txt` globally.

@param [Boolean] mode

@return [Boolean]

@since 0.5.0

# File lib/spidr/spidr.rb, line 34
def self.robots=(mode)
  @robots = mode
end
robots?() click to toggle source

Specifies whether `robots.txt` should be honored globally.

@return [Boolean]

@since 0.5.0

# File lib/spidr/spidr.rb, line 20
def self.robots?
  @robots ||= false
  @robots
end
site(url,**kwargs,&block) click to toggle source

@see Agent.site

# File lib/spidr/spidr.rb, line 64
def self.site(url,**kwargs,&block)
  Agent.site(url,**kwargs,&block)
end
start_at(url,**kwargs,&block) click to toggle source

@see Agent.start_at

# File lib/spidr/spidr.rb, line 41
def self.start_at(url,**kwargs,&block)
  Agent.start_at(url,**kwargs,&block)
end