module ScrapeRequest
@Makes actual scrape request, either raises exception or serves response
Public Class Methods
new(url, read_timeout, connection_timeout)
click to toggle source
# File lib/scraper/scrape_request.rb, line 7 def new(url, read_timeout, connection_timeout) params_hash = { method: :get, url: url, read_timeout: read_timeout, open_timeout: connection_timeout, max_redirects: 10, verify_ssl: false } begin response = RestClient::Request.execute(params_hash) content_type = response.headers[:content_type] return response if content_type =~ %r{(?i)text\s*\/\s*html} exception_message = "Invalid response format received: #{content_type}" raise WebScraper::NonHtmlError, exception_message rescue Zlib::DataError params_hash[:headers] = { 'accept-encoding': 'identity' } retry rescue *TIMEOUT_EXCEPTIONS => e raise WebScraper::TimeoutError, e.message rescue *GENERAL_EXCEPTIONS => e raise WebScraper::RequestError, e.message end end