class Miteru::Crawler

Attributes

downloader[R]
feeds[R]

Public Class Methods

execute() click to toggle source
# File lib/miteru/crawler.rb, line 51
def execute
  new.execute
end
new() click to toggle source
# File lib/miteru/crawler.rb, line 11
def initialize
  @downloader = Downloader.new(Miteru.configuration.download_to)
  @feeds = Feeds.new
  @notifier = Notifier.new
end

Public Instance Methods

auto_download?() click to toggle source
# File lib/miteru/crawler.rb, line 42
def auto_download?
  Miteru.configuration.auto_download?
end
crawl(entry) click to toggle source
# File lib/miteru/crawler.rb, line 17
def crawl(entry)
  website = Website.new(entry.url, entry.source)
  downloader.download_kits(website.kits) if website.has_kits? && auto_download?
  notify(website) if website.has_kits? || verbose?
rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _e
  nil
end
execute() click to toggle source
# File lib/miteru/crawler.rb, line 25
def execute
  suspicious_entries = feeds.suspicious_entries
  puts "Loaded #{suspicious_entries.length} URLs to crawl. (crawling in #{threads} threads)" if verbose?

  Parallel.each(suspicious_entries, in_threads: threads) do |entry|
    crawl entry
  end
end
notify(website) click to toggle source
# File lib/miteru/crawler.rb, line 38
def notify(website)
  @notifier.notify(url: website.url, kits: website.kits, message: website.message)
end
threads() click to toggle source
# File lib/miteru/crawler.rb, line 34
def threads
  @threads ||= Miteru.configuration.threads
end
verbose?() click to toggle source
# File lib/miteru/crawler.rb, line 46
def verbose?
  Miteru.configuration.verbose?
end