class Sunbro::DynamicHTTP
Attributes
session[R]
Public Class Methods
new(opts = {})
click to toggle source
# File lib/sunbro/dynamic_http.rb, line 6 def initialize(opts = {}) @opts = opts Retryable.retryable { new_session } end
Public Instance Methods
close()
click to toggle source
# File lib/sunbro/dynamic_http.rb, line 11 def close @session.driver.quit rescue IOError nil end
fetch_page(url, opts={})
click to toggle source
Create new Pages from the response of an HTTP
request to url, including redirects
# File lib/sunbro/dynamic_http.rb, line 63 def fetch_page(url, opts={}) begin tries ||= 5 get_page(url, opts) rescue IOError, Capybara::Poltergeist::DeadClient, Errno::EPIPE, NoMethodError, Capybara::Poltergeist::BrowserError => e restart_session retry unless (tries -= 1).zero? close raise e end end
get_page(url, opts)
click to toggle source
# File lib/sunbro/dynamic_http.rb, line 75 def get_page(url, opts) reset = opts.fetch(:reset) rescue true start = Time.current.to_i session.visit(url.to_s) page = create_page_from_session(url, session, opts) page.response_time = ((Time.now - start) * 1000).round session.reset! if reset page rescue Capybara::Poltergeist::TimeoutError => e restart_session return Page.new(url, :error => e) end
new_session()
click to toggle source
# File lib/sunbro/dynamic_http.rb, line 17 def new_session Capybara.register_driver :poltergeist do |app| Capybara::Poltergeist::Driver.new( app, timeout: 10, js_errors: false, phantomjs_options: phantomjs_options, ) end Capybara.default_driver = :poltergeist Capybara.javascript_driver = :poltergeist Capybara.run_server = false @session = Capybara::Session.new(:poltergeist) @session.driver.headers = { 'User-Agent' => user_agent } @session end
phantomjs_options()
click to toggle source
# File lib/sunbro/dynamic_http.rb, line 36 def phantomjs_options @phantomjs_options ||= begin opts = [ '--load-images=no', '--ignore-ssl-errors=yes' ] if Sunbro::Settings.proxy_host if Sunbro::Settings.proxy_port opts << "--proxy=#{Sunbro::Settings.proxy_host}:#{Sunbro::Settings.proxy_port}" else opts << "--proxy=#{Sunbro::Settings.proxy_host}" end end opts end end
restart_session()
click to toggle source
# File lib/sunbro/dynamic_http.rb, line 54 def restart_session close Retryable.retryable { new_session } end
user_agent()
click to toggle source
# File lib/sunbro/dynamic_http.rb, line 50 def user_agent @opts[:agent] || Settings.phantomjs_user_agent end
Private Instance Methods
create_page_from_session(url, session, opts)
click to toggle source
# File lib/sunbro/dynamic_http.rb, line 90 def create_page_from_session(url, session, opts) url = url.to_s if url == session.current_url Page.new( session.current_url, :body => session.html.dup, :code => session.status_code, :headers => session.response_headers, :force_format => (opts[:force_format] || default_page_format) ) else Page.new( session.current_url, :body => session.html.dup, :code => 301, :redirect_from => url, :headers => session.response_headers, :force_format => (opts[:force_format] || default_page_format) ) end end
default_page_format()
click to toggle source
# File lib/sunbro/dynamic_http.rb, line 112 def default_page_format # Don't force the page format if the default format is set to :any return unless [:xml, :html].include? Settings.page_format Settings.page_format end