class BrowserCrawler::Reports::Store
It involves methods which allow to save data to a store structure
Attributes
crawler_error[R]
error[RW]
metadata[R]
pages[R]
unrecognized_links[R]
Public Class Methods
new(pages: {}, metadata: {}, started_at: nil, finished_at: nil)
click to toggle source
# File lib/browser_crawler/reports/store.rb, line 39 def initialize(pages: {}, metadata: {}, started_at: nil, finished_at: nil) @pages = pages @metadata = metadata @started_at = started_at @finished_at = finished_at @crawler_error = {} @unrecognized_links = [] end
Public Instance Methods
finish()
click to toggle source
# File lib/browser_crawler/reports/store.rb, line 57 def finish @finished_at = Time.now end
record_crawler_error(link:, error:)
click to toggle source
# File lib/browser_crawler/reports/store.rb, line 94 def record_crawler_error(link:, error:) @crawler_error[link] = { message: error.message, backtrace: error.backtrace } end
record_page_visit(page:, extracted_links: nil, screenshot_filename: nil, error: nil, external: false, code: nil)
click to toggle source
# File lib/browser_crawler/reports/store.rb, line 79 def record_page_visit(page:, extracted_links: nil, screenshot_filename: nil, error: nil, external: false, code: nil) @pages[page] = { screenshot: screenshot_filename, error: error, extracted_links: extracted_links, code: code, external: external } end
record_unrecognized_link(link)
click to toggle source
# File lib/browser_crawler/reports/store.rb, line 73 def record_unrecognized_link(link) return if @unrecognized_links.include?(link) @unrecognized_links << link unless @unrecognized_links.include?(link) end
start(url:)
click to toggle source
# File lib/browser_crawler/reports/store.rb, line 51 def start(url:) @pages.clear @started_at = Time.now @metadata[:url] = url end
to_h()
click to toggle source
# File lib/browser_crawler/reports/store.rb, line 61 def to_h {}.merge(pages: @pages) .merge(@metadata) .merge( unrecognized_links: @unrecognized_links, crawler_error: @crawler_error, started_at: @started_at, finished_at: @finished_at, links_count: count_all_links ) end
visited_pages()
click to toggle source
# File lib/browser_crawler/reports/store.rb, line 101 def visited_pages @pages.keys end
Private Instance Methods
count_all_links()
click to toggle source
# File lib/browser_crawler/reports/store.rb, line 107 def count_all_links @pages.inject(0) do |sum, (_, data)| sum + data[:extracted_links]&.size.to_i if data && data[:extracted_links] end end