class DomainsScanner::Crawlers::Base

Public Instance Methods

agent() click to toggle source
# File lib/domains_scanner/crawlers/base.rb, line 6
def agent
  @agent ||= Mechanize.new do |agent|
    agent.user_agent_alias = "Mac Safari"
  end
end
have_next_page?(doc) click to toggle source
# File lib/domains_scanner/crawlers/base.rb, line 54
def have_next_page?(doc)
  raise NotImplementedError
end
keyword_field_name() click to toggle source
# File lib/domains_scanner/crawlers/base.rb, line 46
def keyword_field_name
  raise NotImplementedError
end
parse_results(doc) click to toggle source
# File lib/domains_scanner/crawlers/base.rb, line 50
def parse_results(doc)
  raise NotImplementedError
end
search_by_form(domain_name, top_level_domain) click to toggle source
# File lib/domains_scanner/crawlers/base.rb, line 12
def search_by_form(domain_name, top_level_domain)
  doc = agent.get(host)

  form = doc.forms.first
  query = search_keyword(domain_name, top_level_domain)
  form[keyword_field_name] = query
  doc = form.submit

  results = parse_results(doc)
  next_page_link = parse_next_page_link(doc)

  DomainsScanner::Results.new(results, next_page_link)
end
search_keyword(domain_name, top_level_domain) click to toggle source
# File lib/domains_scanner/crawlers/base.rb, line 42
def search_keyword(domain_name, top_level_domain)
  "site:*.#{domain_name}.#{top_level_domain}"
end