class Popularity::Crawler
Attributes
url[R]
Public Class Methods
new(url)
click to toggle source
# File lib/popularity/crawler.rb, line 26 def initialize(url) @url = url end
property_names()
click to toggle source
# File lib/popularity/crawler.rb, line 18 def self.property_names @property_names end
stats(*args)
click to toggle source
# File lib/popularity/crawler.rb, line 11 def self.stats(*args) @property_names ||= [] args.each do |name| @property_names << name end end
Public Instance Methods
as_json(options = {})
click to toggle source
# File lib/popularity/crawler.rb, line 67 def as_json(options = {}) json = {} self.class.property_names.each do |name| json[name.to_s] = self.send(name.to_sym) end json["total"] = total json end
async_done?()
click to toggle source
# File lib/popularity/crawler.rb, line 51 def async_done? @async_done end
fetch()
click to toggle source
# File lib/popularity/crawler.rb, line 88 def fetch return false unless valid? begin response = Unirest.get(request_url) @response = response.raw_body rescue OpenURI::HTTPError, Timeout::Error, SocketError nil end end
fetch_async(&block)
click to toggle source
# File lib/popularity/crawler.rb, line 78 def fetch_async(&block) return false unless valid? Unirest.get(request_url) do |response| @async_done = true @response = response.raw_body block.call(response.code, response.raw_body) if block_given? end end
has_response?()
click to toggle source
# File lib/popularity/crawler.rb, line 38 def has_response? response #fetch it return false if response.nil? return false if response.empty? true end
host()
click to toggle source
# File lib/popularity/crawler.rb, line 55 def host URI.parse(@url).host.gsub('www.', '') end
name()
click to toggle source
# File lib/popularity/crawler.rb, line 63 def name self.class.to_s.split('::').last.gsub(/(.)([A-Z])/,'\1_\2').downcase end
request_url()
click to toggle source
# File lib/popularity/crawler.rb, line 30 def request_url raise NotImplemented end
response()
click to toggle source
# File lib/popularity/crawler.rb, line 34 def response @response ||= fetch end
response_json()
click to toggle source
# File lib/popularity/crawler.rb, line 59 def response_json @json ||= JSON.parse(response) end
total()
click to toggle source
# File lib/popularity/crawler.rb, line 22 def total self.class.property_names.uniq.collect { |n| self.send(n.to_sym) }.select { |t| t.class == Fixnum }.compact.reduce(:+) end
valid?()
click to toggle source
# File lib/popularity/crawler.rb, line 47 def valid? true # to be overridden in subclasses end