class DomainsScanner::Crawlers::Base
Public Instance Methods
agent()
click to toggle source
# File lib/domains_scanner/crawlers/base.rb, line 6 def agent @agent ||= Mechanize.new do |agent| agent.user_agent_alias = "Mac Safari" end end
have_next_page?(doc)
click to toggle source
# File lib/domains_scanner/crawlers/base.rb, line 54 def have_next_page?(doc) raise NotImplementedError end
keyword_field_name()
click to toggle source
# File lib/domains_scanner/crawlers/base.rb, line 46 def keyword_field_name raise NotImplementedError end
parse_next_page_link(doc)
click to toggle source
# File lib/domains_scanner/crawlers/base.rb, line 34 def parse_next_page_link(doc) next_page_tag = doc.search(next_page_link_selector).first return unless next_page_tag href = next_page_tag.attributes["href"] "#{host}#{href}" end
parse_results(doc)
click to toggle source
# File lib/domains_scanner/crawlers/base.rb, line 50 def parse_results(doc) raise NotImplementedError end
search_by_form(domain_name, top_level_domain)
click to toggle source
# File lib/domains_scanner/crawlers/base.rb, line 12 def search_by_form(domain_name, top_level_domain) doc = agent.get(host) form = doc.forms.first query = search_keyword(domain_name, top_level_domain) form[keyword_field_name] = query doc = form.submit results = parse_results(doc) next_page_link = parse_next_page_link(doc) DomainsScanner::Results.new(results, next_page_link) end
search_by_link(link)
click to toggle source
# File lib/domains_scanner/crawlers/base.rb, line 26 def search_by_link(link) doc = agent.get(link) results = parse_results(doc) next_page_link = parse_next_page_link(doc) DomainsScanner::Results.new(results, next_page_link) end
search_keyword(domain_name, top_level_domain)
click to toggle source
# File lib/domains_scanner/crawlers/base.rb, line 42 def search_keyword(domain_name, top_level_domain) "site:*.#{domain_name}.#{top_level_domain}" end