class AutoServiceCLI::Scraper
Constants
- DEFAULT_SORT_TYPE
- SORT_TYPES
Attributes
sort_type[RW]
zip[RW]
Public Class Methods
new(zip = nil, sort_type = nil)
click to toggle source
Constructors and class methods.
# File lib/auto_service_cli/scraper.rb, line 16 def initialize(zip = nil, sort_type = nil) self.zip = zip if zip self.sort_type = sort_type ? sort_type : self.class::SORT_TYPES[:DEFAULT] end
valid_sort_type?(type)
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 33 def self.valid_sort_type?(type) SORT_TYPES.any? { |key, value| value == type } end
valid_zip?(zip)
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 25 def self.valid_zip?(zip) zip.to_i > 0 end
Public Instance Methods
scrape_center_details(center)
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 71 def scrape_center_details(center) raise InvalidPage, "Invalid page!!!" if !center.is_a? AutoServiceCLI::ServiceCenter || center.int_url.nil? doc = Nokogiri::HTML(open(center.int_url)) raise InvalidPage "Invalid page!!!" if doc.nil? details = {} status = scrape_status(doc); details[:open_status] = status unless status.nil? slogan = scrape_slogan(doc); details[:slogan] = slogan unless slogan.nil? hours = scrape_hours(doc); details[:working_hours] = hours unless hours.nil? description = scrape_description(doc); details[:description] = description unless description.nil? services = scrape_services(doc); details[:services] = services unless services.nil? brands = scrape_brands(doc); details[:brands] = brands unless brands.nil? payment = scrape_payment(doc); details[:payment] = payment unless payment.nil? center.details_from_hash(details) end
scrape_centers()
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 45 def scrape_centers doc = Nokogiri::HTML(open(get_url)) raise InvalidPage "Invalid page!!!" if doc.nil? centers = doc.css(".organic .result .info") # form centers hash. centers.each do |center| main_details = {} obj_center = AutoServiceCLI::ServiceCenter.create(scrape_name(center)) url = scrape_internal_url(center); main_details[:int_url] = url unless url.nil? url = scrape_external_url(center); main_details[:ext_url] = url unless url.nil? rating = scrape_rating(center); main_details[:rating] = rating unless rating.nil? address = scrape_address(center); main_details[:address] = address unless address.nil? phone = scrape_phone_number(center); main_details[:phone_number] = phone unless phone.nil? category = scrape_category(center); main_details[:main_category] = category unless category.nil? obj_center.details_from_hash(main_details) # setting all details to center. end end
sort_type=(type)
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 29 def sort_type=(type) raise InvalidURLData unless self.class.valid_sort_type?(type) @sort_type = type end
zip=(zip)
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 21 def zip=(zip) raise InvalidURLData unless self.class.valid_zip?(zip) @zip = zip end
Private Instance Methods
external_link?(url)
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 99 def external_link?(url) url.include?("www.") || url.include?("http://") || url.include?("https://") end
get_url()
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 94 def get_url raise InvalidURLData if !self.zip || !self.sort_type AutoServiceCLI::URL_TEMPLATE + "#{self.zip}&s=#{self.sort_type}" end
scrape_address(center)
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 127 def scrape_address(center) address_full = center.css(".info-primary .adr span").collect {|el| el.text.split(",").first} address_full.flatten.empty? ? nil : address_full.join(", ") end
scrape_brands(doc)
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 180 def scrape_brands(doc) brands = doc.css("#business-info .brands") brands.empty? ? nil : brands.text end
scrape_category(center)
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 137 def scrape_category(center) category = center.css(".info-secondary .categories a") category.empty? ? nil : category.text end
scrape_description(doc)
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 163 def scrape_description(doc) description = doc.css("#business-info .general-info p").last # description = doc.css("#business-info dd.description").last if description.nil? (!description.nil? && description.css("a").empty?) ? description.text : nil #exclude case of facebook and twitter links, without decription end
scrape_external_url(center)
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 112 def scrape_external_url(center) url = center.css(".links a.track-visit-website") url.empty? ? nil : url.attr("href").value end
scrape_hours(doc)
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 153 def scrape_hours(doc) working_hours = "" doc.css("#business-info .open-hours time").each do |time| working_hours << time.css(".day-label").text working_hours << ", " + time.css(".day-hours").text + "\n" end working_hours.empty? ? nil : working_hours end
scrape_internal_url(center)
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 107 def scrape_internal_url(center) url = center.css(".n a").attr("href").value (!url.empty? && !external_link?(url)) ? AutoServiceCLI::URL_BASE + url : nil end
scrape_name(center)
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 103 def scrape_name(center) center.css(".n a").text end
scrape_payment(doc)
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 185 def scrape_payment(doc) payment = doc.css("#business-info .payment") payment.empty? ? nil : payment.text end
scrape_phone_number(center)
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 132 def scrape_phone_number(center) phone_number = center.css(".info-primary .phone.primary") phone_number.empty? ? nil : phone_number.text end
scrape_rating(center)
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 117 def scrape_rating(center) rate = center.css(".info-primary .result-rating") unless rate.empty? attributes = rate.attr("class").value.split(" "); attributes.slice(1, attributes.size - 1) else nil end end
scrape_services(doc)
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 169 def scrape_services(doc) for_services = doc.css("#business-info dl")[1] unless for_services.nil? if for_services.css("dt").first.text == "Services/Products:" services = for_services.css("dd").first.text return services.empty? ? nil : services end end nil end
scrape_slogan(doc)
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 147 def scrape_slogan(doc) slogan = doc.css("#business-info .slogan") puts slogan slogan.empty? ? nil : slogan.text end
scrape_status(doc)
click to toggle source
# File lib/auto_service_cli/scraper.rb, line 142 def scrape_status(doc) open_status = doc.css(".business-card-wrapper .status-text") open_status.empty? ? nil : open_status.text end