class WatirCrawler::Base
Attributes
timeouts[R]
Public Class Methods
new(timeouts = {})
click to toggle source
# File lib/watir_crawler/base.rb, line 7 def initialize(timeouts = {}) @elements_path = [] @timeouts = { :wait_timeout => 150 }.merge(timeouts) @browser = WatirCrawler::Browser.new(@timeouts) end
Public Instance Methods
browser()
click to toggle source
# File lib/watir_crawler/base.rb, line 19 def browser @browser.browser end
browser_profile() { |profile| ... }
click to toggle source
# File lib/watir_crawler/base.rb, line 13 def browser_profile @browser.profile do |profile| yield profile if block_given? end end
browser_session() { || ... }
click to toggle source
# File lib/watir_crawler/base.rb, line 31 def browser_session timer do catch_error do browser_start yield end end ensure browser_stop end
browser_start()
click to toggle source
# File lib/watir_crawler/base.rb, line 23 def browser_start @browser.start end
browser_stop()
click to toggle source
# File lib/watir_crawler/base.rb, line 27 def browser_stop @browser.stop end
catch_error() { || ... }
click to toggle source
# File lib/watir_crawler/base.rb, line 50 def catch_error yield rescue Timeout::Error, # http connection with driver Selenium::WebDriver::Error::TimeOutError, # browser.driver.manage.timeouts.page_load Selenium::WebDriver::Error::ScriptTimeOutError # browser.driver.manage.timeouts.script_timeout log.error "Site is too slow at page: '#{browser.url}'" raise SiteTooSlow rescue SystemCallError, # 'Unknown error - Connection reset by peer' Errno::ECONNREFUSED, # 'Connection refused - Connection refused' Selenium::WebDriver::Error::WebDriverError => e # 'unable to obtain stable firefox connection in 60 seconds (127.0.0.1:7055)' # 'unable to bind to locking port 7054 within 45 seconds' messages = [ /Connection reset by peer/, # SystemCallError /Connection refused/, # Errno::ECONNREFUSED /unable to obtain stable firefox connection/, # Selenium::WebDriver::Error::WebDriverError /unable to bind to locking port/ # Selenium::WebDriver::Error::WebDriverError ] log "#{e.class}: #{e.message} \n#{e.backtrace.join("\n")}" klass = messages.select{|msg| msg =~ e.message }.any? ? WebdriverError : SiteChanged raise klass end
common_wait(*args, &block)
click to toggle source
# File lib/watir_crawler/base.rb, line 140 def common_wait *args, &block browser.wait_until(@timeouts[:wait_timeout]) do #todo 1 raise_if_firefox_error if respond_to?(:raise_if_firefox_error) #todo 2 raise_if_service_unavailable if respond_to?(:raise_if_service_unavailable) # see class method :raise_service_unavailable_if if args.any? || block pull(args) || (block && instance_eval(&block)) else return nil # running raise_if 1 times and exit if no args & block end end rescue Selenium::WebDriver::Error::StaleElementReferenceError, Selenium::WebDriver::Error::ObsoleteElementError sleep 1 retry rescue Watir::Wait::TimeoutError raise SiteChanged end
exec(script)
click to toggle source
# File lib/watir_crawler/base.rb, line 82 def exec script browser.execute_script(script) end
exist?(xpath)
click to toggle source
# File lib/watir_crawler/base.rb, line 160 def exist? xpath !!pull(xpath) end
goto(url)
click to toggle source
— commands
# File lib/watir_crawler/base.rb, line 78 def goto url browser.goto url if url != browser.url end
pull(*args) { || ... }
click to toggle source
# File lib/watir_crawler/base.rb, line 88 def pull *args, &block opts, xpaths = args.flatten.partition{|a| a.is_a?(Symbol) } opt_mode = opts.delete(:exist?) || opts.delete(:present?) || :present? # default is :present? opt_first = !!opts.delete(:first) || !opts.delete(:all) # default is true, return 1th element raise "Unknown options: '#{opts.inspect}'" if opts.any? elements = xpaths.select do |xpath| node_for(xpath).send(opt_mode) # detect element on the page by opt_mode end.map do |xpath| nodes_for(xpath) # get all elements end.flatten.select do |node| node.send(opt_mode) # select elements by mode end.map do |node| subtype = node.to_subtype class << subtype attr_accessor :node_xpath end subtype.node_xpath = node.node_xpath subtype end # flash result nodes elements = elements.take(1) if opt_first elements.each{|node| node.flash unless node.is_a?(Watir::Frame) || node.is_a?(Watir::IFrame) } first_element = elements.first if block raise SiteChanged, "Not found elements for xpath: #{xpaths.inspect}" if first_element.nil? nodes_path << first_element.node_xpath yield else opt_first ? first_element : elements end rescue Selenium::WebDriver::Error::StaleElementReferenceError, Selenium::WebDriver::Error::ObsoleteElementError sleep 1 retry ensure nodes_path.pop if block end
timer() { || ... }
click to toggle source
# File lib/watir_crawler/base.rb, line 42 def timer log.info "Session start" start_time = Time.now yield ensure log.info "Session end, elapsed time: #{Time.now - start_time}" end
wait(*xpaths, &block)
click to toggle source
:first - get FIRST element of FIRST founded xpath, DEFAULT OPTION :all - get ALL elements of FIRST founded xpath
# File lib/watir_crawler/base.rb, line 134 def wait *xpaths, &block #todo 3 raise_if_site_too_slow if respond_to?(:raise_if_site_too_slow) common_wait *xpaths, &block end
Private Instance Methods
element_name_for(xpath, plural = false)
click to toggle source
# File lib/watir_crawler/base.rb, line 205 def element_name_for xpath, plural = false [ xpath_with_frame?(xpath) ? 'frame' : 'element', plural ? 's' : '', '(:xpath, "' + xpath + '")' ].join end
get_nodes(xpath, opts)
click to toggle source
# File lib/watir_crawler/base.rb, line 180 def get_nodes xpath, opts element_path = xpath_relative?(xpath) ? nodes_path.map { |node_xpath| element_name_for(node_xpath) } : nil element_name = element_name_for(xpath, opts[:get_all]) eval_string = [element_path, element_name].flatten.compact.join('.') log "#{File.basename(__FILE__)}:#{__LINE__}, eval_string: " + eval_string.inspect elements = browser.instance_eval(eval_string) elements = elements.to_a if elements.is_a? Watir::ElementCollection elements = [elements].flatten has_multiple_elements = elements.size > 1 elements.map.with_index do |element, index| class << element attr_accessor :node_xpath end element.node_xpath = xpath element.node_xpath += "[#{index + 1}]" if has_multiple_elements element end end
node_for(xpath)
click to toggle source
# File lib/watir_crawler/base.rb, line 172 def node_for xpath get_nodes(xpath, :get_all => false).first end
nodes_for(xpath)
click to toggle source
# File lib/watir_crawler/base.rb, line 176 def nodes_for xpath get_nodes(xpath, :get_all => true) end
nodes_path()
click to toggle source
# File lib/watir_crawler/base.rb, line 168 def nodes_path @nodes_path ||= [] end
xpath_relative?(xpath)
click to toggle source
# File lib/watir_crawler/base.rb, line 213 def xpath_relative? xpath xpath =~ /^\.\/.*/ # "./" end
xpath_with_frame?(xpath)
click to toggle source
# File lib/watir_crawler/base.rb, line 217 def xpath_with_frame? xpath xpath =~ /^[\.]?\/[\/]?[i]?frame.*/ # "//frame", "//iframe", ".//frame", ".//iframe" end