class Miteru::Crawler
Attributes
downloader[R]
feeds[R]
Public Class Methods
execute()
click to toggle source
# File lib/miteru/crawler.rb, line 51 def execute new.execute end
new()
click to toggle source
# File lib/miteru/crawler.rb, line 11 def initialize @downloader = Downloader.new(Miteru.configuration.download_to) @feeds = Feeds.new @notifier = Notifier.new end
Public Instance Methods
auto_download?()
click to toggle source
# File lib/miteru/crawler.rb, line 42 def auto_download? Miteru.configuration.auto_download? end
crawl(entry)
click to toggle source
# File lib/miteru/crawler.rb, line 17 def crawl(entry) website = Website.new(entry.url, entry.source) downloader.download_kits(website.kits) if website.has_kits? && auto_download? notify(website) if website.has_kits? || verbose? rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _e nil end
execute()
click to toggle source
# File lib/miteru/crawler.rb, line 25 def execute suspicious_entries = feeds.suspicious_entries puts "Loaded #{suspicious_entries.length} URLs to crawl. (crawling in #{threads} threads)" if verbose? Parallel.each(suspicious_entries, in_threads: threads) do |entry| crawl entry end end
notify(website)
click to toggle source
# File lib/miteru/crawler.rb, line 38 def notify(website) @notifier.notify(url: website.url, kits: website.kits, message: website.message) end
threads()
click to toggle source
# File lib/miteru/crawler.rb, line 34 def threads @threads ||= Miteru.configuration.threads end
verbose?()
click to toggle source
# File lib/miteru/crawler.rb, line 46 def verbose? Miteru.configuration.verbose? end