class WIKK::WebBrowser

WIKK WebBrowser class under MIT Lic. github.com/wikarekare. Wrapper around ruby's http classes WIKK_WebBrowser.new.https_session(host: 'www.blah.com') do |session| response = get_page(query: ,'/') end

Constants

VERSION

Attributes

cookies[RW]
debug[RW]
host[R]
page[R]
port[RW]
referer[RW]
response[RW]
session[RW]
use_ssl[RW]
verify_cert[RW]

Public Class Methods

http_session(host:, port: nil, debug: false, cookies: {}) { |wb| ... } click to toggle source

Create a WIKK_WebBrowser instance, connect to the host via http, and yield the WIKK_WebBrowser instance. Automatically closes the http session on returning from the block passed to it.

@param host [String] the host we want to connect to

@param port [Fixnum] (80) the port the remote web server is running on

@param block [Proc]

@yieldparam [WIKK_WebBrowser] the session descriptor for further calls.

# File lib/wikk_webbrowser.rb, line 69
def self.http_session(host:, port: nil, debug: false, cookies: {})
  wb = self.new(host: host, port: port, debug: debug, use_ssl: false, cookies: cookies)
  wb.http_session do
    yield wb
  end
end
https_session(host:, port: nil, verify_cert: true, cookies: {}, debug: false) { |wb| ... } click to toggle source

Create a WIKK_WebBrowser instance, connect to the host via https, and yield the WIKK_WebBrowser instance. Automatically closes the http session on returning from the block passed to it. @param host [String] the host we want to connect to @param port [Fixnum] (443) the port the remote web server is running on @param verify_cert [Boolean] Validate certificate if true (Nb lots of embedded devices have self signed certs, so verify will fail) @param block [Proc] @yieldparam [WIKK_WebBrowser] the session descriptor for further calls.

# File lib/wikk_webbrowser.rb, line 83
def self.https_session(host:, port: nil, verify_cert: true, cookies: {}, debug: false)
  wb = self.new(host: host, port: port, cookies: cookies, use_ssl: true, verify_cert: verify_cert, debug: debug)
  wb.http_session do
    yield wb
  end
end
new(host:, port: nil, use_ssl: false, cookies: {}, verify_cert: true, debug: false) click to toggle source

Create a WIKK_WebBrowser instance

@param host [String] the host we want to connect to @param port [Fixnum] Optional http server port @param use_ssl [Boolean] Use https, if true @param verify_cert [Boolean] Validate certificate if true (Nb lots of embedded devices have self signed certs, so verify will fail) @return [WIKK_WebBrowser]

# File lib/wikk_webbrowser.rb, line 48
def initialize(host:, port: nil, use_ssl: false, cookies: {}, verify_cert: true, debug: false)
  @host = host  #Need to do this, as passing nil is different to passing nothing to initialize!
  @cookies = cookies == nil ? {} : cookies
  @debug = debug
  @use_ssl = use_ssl
  @port = port != nil ? port : ( use_ssl ? 443 : 80 )
  @verify_cert = verify_cert
  @response = nil
end

Public Instance Methods

add_cookies(cookies) click to toggle source

Add additional cookies @param cookies [Hash] cookie_name => cookie_value

# File lib/wikk_webbrowser.rb, line 125
def add_cookies(cookies)
  cookies.each { |cookie_name, cookie_value| @cookies[cookie_name] = cookie_value }
end
basic_authorization(user:, password:) click to toggle source

Web basic authentication (not exactly secure) @param user [String] Account name @param password [String] Accounts password @return [String] Base64 encoded concatentation of user + ':' + password

# File lib/wikk_webbrowser.rb, line 111
def basic_authorization(user:, password:)
  #req.basic_auth( user, password) if  user != nil
  'Basic ' + Base64.encode64( "#{user}:#{password}" )
end
bearer_authorization(token:) click to toggle source

Dropbox style token authentication @param token [String] Token, as issued by dropbox @return [String] Concatenation of 'Bearer ' + token

# File lib/wikk_webbrowser.rb, line 119
def bearer_authorization(token:)
  "Bearer " + token
end
cookies_to_s() click to toggle source

Convert @cookies to ; separated strings @return cookies string

# File lib/wikk_webbrowser.rb, line 143
def cookies_to_s
  @cookies.to_a.map { |v| v.join('=') }.join('; ')
end
delete_req(query:, authorization: nil, extra_headers: {}, extra_cookies: {}) click to toggle source

send a DELETE query to the server and return the response. @param query [String] URL, less the 'host/' part @param authorization [String] If present, add Authorization header, using this string @param extra_headers [Hash] Add these to standard headers @param extra_cookies [Hash] Add these to standard cookies @return [String] The Net::HTTPResponse.body text response from the web server

# File lib/wikk_webbrowser.rb, line 278
def delete_req(query:, authorization: nil, extra_headers: {}, extra_cookies: {})
  url = URI.parse("#{@use_ssl ? "https" : "http"}://#{@host}/#{query.gsub(/^\//,'')}")
  req = Net::HTTP::Delete.new(query)

  header = { 'HOST' => @host }
  add_cookies(extra_cookies)
  header['Cookie'] = cookies_to_s if @cookies.length > 0
  header['Authorization'] = authorization if authorization != nil

  extra_headers.each do |k,v| 
    header[k] = v
  end
  req.initialize_http_header( header )

  begin
    @response = @session.request(req)
    save_cookies(@response)
  
    if(@response.code.to_i >= 300)
      raise "#{url} : #{@response.code} #{@response.message}"
    end
    return @response.body
  rescue StandardError => e
    puts "#{e}"
    return nil
  end
end
extract_input_fields(body) click to toggle source

Extract form field values from the html body. @param body [String] The html response body @return [Hash] Keys are the field names, values are the field values

# File lib/wikk_webbrowser.rb, line 369
def extract_input_fields(body)
  entry = true
  @inputs = {}
  doc = Nokogiri::HTML(body)
  doc.xpath("//form/input").each do |f|
    @inputs[f.get_attribute('name')] = f.get_attribute('value')
  end
end
form_values_to_s(form_values=nil, has_q = false) click to toggle source

Take a hash of the params to the post and generate a safe URL string. @param form_values [Hash] Keys are the field names, values are the field values @param has_q [Boolean] We have a leading ? for the html get, so don't need to add one. @return [String] The 'safe' text for fields the get or post query to the web server

# File lib/wikk_webbrowser.rb, line 395
def form_values_to_s(form_values=nil, has_q = false)
  return "" if form_values == nil || form_values.length == 0
  s = (has_q == true ? "" : "?")
  first = true
  form_values.each do |key,value|
    s += "&" if !first
    s += "#{CGI::escape(key.to_s)}=#{CGI::escape(value.to_s)}"
    first = false
  end
  return s
end
get_page(query: ,form_values: nil, authorization: nil, extra_headers: {}, extra_cookies: {}) click to toggle source

send a GET query to the web server using an http get, and returns the response. Cookies in the response get preserved in @cookies, so they will be sent along with subsequent calls We are currently ignoring redirects from the PDU's we are querying. @param query [String] The URL after the host/ bit and not usually not including parameters, if form_values are passed in @param form_values [Hash{String=>Object-with-to_s}] The parameter passed to the web server eg. ?key1=value1&key2=value2… @param authorization [String] If present, add Authorization header, using this string @param extra_headers [Hash] Add these to standard headers @param extra_cookies [Hash] Add these to standard cookies @return [String] The Net::HTTPResponse.body text response from the web server

# File lib/wikk_webbrowser.rb, line 164
def get_page(query: ,form_values: nil, authorization: nil, extra_headers: {}, extra_cookies: {})
  $stderr.puts "Debugging On" if @debug
  query += form_values_to_s(form_values, query.index('?') != nil) #Should be using req.set_form_data, but it seems to by stripping the leading / and then the query fails.
  url = URI.parse("#{@use_ssl ? "https" : "http"}://#{@host}/#{query.gsub(/^\//,'')}")
  $stderr.puts url if @debug

  req = Net::HTTP::Get.new(url.request_uri)  

  header = { 'HOST' => @host }
  header['Accept'] = '*/*'
  header['Accept-Encoding'] = 'gzip, deflate, br'
  header['Accept-Language'] = 'en-US,en;q=0.5'
  header['Connection'] = 'keep-alive'
  header['User-Agent'] = 'Mozilla/5.0'
  header['Content-Type'] = 'application/x-www-form-urlencoded'   
  add_cookies(extra_cookies)
  header['Cookie'] = cookies_to_s if @cookies.length > 0
  header['DNT'] = "1"
  header['Authorization'] = authorization if authorization != nil

  extra_headers.each do |k,v| 
    header[k] = v
  end

  req.initialize_http_header( header )

  @response = @session.request(req)      
  save_cookies(@response)

  $stderr.puts @response.code.to_i if @debug

  if(@response.code.to_i >= 300)
    if(@response.code.to_i == 302)
        #ignore the redirects.
        #$stderr.puts "302"
        #@response.each {|key, val| $stderr.printf "%s = %s\n", key, val }  #Location seems to have cgi params removed. End up with .../cginame?&
        #$stderr.puts "Redirect to #{@response['location']}"   #Location seems to have cgi params removed. End up with .../cginame?&
        #$stderr.puts
      return
    elsif @response.code.to_i >= 400 && @response.code.to_i < 500
      return @response.body
    end
    raise Error.new(web_return_code: @response.code.to_i, message: "#{@response.code} #{@response.message} #{query} #{form_values} #{@response.body}")
  end

  return @response.body
end
header_value(key:) click to toggle source

Get a header value, from the last response

@param key [String] header key @return [String] header value, for the given key.

# File lib/wikk_webbrowser.rb, line 151
def header_value(key:)
  @response.header[key]
end
http_session() { || ... } click to toggle source

Creating a session for http connection attached block would then call get or post NET::HTTP calls @param port [Fixnum] Optional http server port @param use_ssl [Boolean] Use https, if true @param verify_cert [Boolean] Validate certificate if true (Nb lots of embedded devices have self signed certs, so verify will fail) @param block [Proc]

# File lib/wikk_webbrowser.rb, line 96
def http_session
  @http = Net::HTTP.new(@host, @port)   
  @http.set_debug_output($stdout) if @debug
  @http.use_ssl = @use_ssl      
  @http.verify_mode = OpenSSL::SSL::VERIFY_NONE  if ! @use_ssl || ! @verify_cert 
  @http.start do |session| #ensure we close the session after the block
    @session = session 
    yield 
  end
end
post_page(query:, authorization: nil, content_type: 'application/x-www-form-urlencoded', data: nil, extra_headers: {}, extra_cookies: {}) click to toggle source

send a POST query to the server and return the response. @param query [String] URL, less the 'host/' part @param authorization [String] If present, add Authorization header, using this string @param content_type [String] Posted content type @param data [String] Text to add to body of post to the web server @param extra_headers [Hash] Add these to standard headers @param extra_cookies [Hash] Add these to standard cookies @return [String] The Net::HTTPResponse.body text response from the web server

# File lib/wikk_webbrowser.rb, line 220
def post_page(query:, authorization: nil, content_type: 'application/x-www-form-urlencoded', data: nil, extra_headers: {}, extra_cookies: {})
  url = URI.parse("#{@use_ssl ? "https" : "http"}://#{@host}/#{query}")
  req = Net::HTTP::Post.new(url.path)

  header = { 'HOST' => @host }
  header['Accept'] = '*/*'
  header['Accept-Encoding'] = 'gzip, deflate, br'
  header['Accept-Language'] = 'en-US,en;q=0.5'
  header['Connection'] = 'keep-alive'
  header['User-Agent'] = 'Mozilla/5.0'
  header['Content-Type'] = content_type
  add_cookies(extra_cookies)
  header['Cookie'] = cookies_to_s if @cookies.length > 0
  header['DNT'] = "1"
  header['Authorization'] = authorization if authorization != nil

  extra_headers.each do |k,v| 
    header[k] = v
  end
  req.initialize_http_header( header )

  if data != nil
    if data.class == Hash
      if content_type =~ /application\/octet-stream/
        req.set_form_data(data, '&') 
      else
        req.set_form_data.to_j
      end
    else
      req.body = data #If json as a string or raw string
    end
  else
    req.body = ''
  end
  
  @response = @session.request(req)
  save_cookies(@response)

  if(@response.code.to_i >= 300)
    if(@response.code.to_i == 302)
        #ignore the redirects.
        #puts "302"
        #@response.each {|key, val| printf "%s = %s\n", key, val }  #Location seems to have cgi params removed. End up with .../cginame?&
        #puts "Redirect of Post to #{@response['location']}" #Location seems to have cgi params removed. End up with .../cginame?&
      return
    end
    raise Error.new(web_return_code: @response.code, message: "#{@response.code} #{@response.message} #{query} #{data} #{@response.body}")
  end
  
  return @response.body
end
put_req(query:, authorization: nil, content_type: '"application/octet-stream"', data: nil, extra_headers: {}, extra_cookies: {}) click to toggle source

send a PUT query to the server and return the response. @param query [String] URL, less the 'host/' part @param authorization [String] If present, add Authorization header, using this string @param content_type [String] Posted content type @param data [String] Text to add to body of post to the web server @param extra_headers [Hash] Add these to standard headers @param extra_cookies [Hash] Add these to standard cookies @return [String] The Net::HTTPResponse.body text response from the web server

# File lib/wikk_webbrowser.rb, line 314
def put_req(query:, authorization: nil, content_type: '"application/octet-stream"', data: nil, extra_headers: {}, extra_cookies: {})
  url = URI.parse("#{@use_ssl ? "https" : "http"}://#{@host}/#{query}")
  req = Net::HTTP::Put.new(url.path)

  header = { 'HOST' => @host }
  header['Accept'] = '*/*'
  header['Accept-Encoding'] = 'gzip, deflate, br'
  header['Accept-Language'] = 'en-US,en;q=0.5'
  header['Connection'] = 'keep-alive'
  header['User-Agent'] = 'Mozilla/5.0'
  header['Content-Type'] = content_type
  add_cookies(extra_cookies)
  header['Cookie'] = cookies_to_s if @cookies.length > 0
  header['DNT'] = "1"
  header['Authorization'] = authorization if authorization != nil

  extra_headers.each do |k,v| 
    header[k] = v
  end
  req.initialize_http_header( header )

  if data != nil
    if data.class == Hash
      if content_type =~ /application\/octet-stream/
        req.set_form_data(data, '&') 
      else
        req.set_form_data.to_j
      end
    else
      req.body = data #If json as a string or raw string
    end
  else
    req.body = ''
  end
  
  @response = @session.request(req)
  save_cookies(@response)

  if(@response.code.to_i >= 300)
    if(@response.code.to_i == 302)
        #ignore the redirects.
        #puts "302"
        #@response.each {|key, val| printf "%s = %s\n", key, val }  #Location seems to have cgi params removed. End up with .../cginame?&
        #puts "Redirect of Post to #{@response['location']}" #Location seems to have cgi params removed. End up with .../cginame?&
      return
    end
    raise Error.new(web_return_code: @response.code, message: "#{@response.code} #{@response.message} #{query} #{data} #{@response.body}")
  end
  
  return @response.body
end
save_cookies(response) click to toggle source

Save cookies returned by last html get/post. Removes previous cookies. @param response [Net::HTTPResponse] result from HTTP calls

# File lib/wikk_webbrowser.rb, line 132
def save_cookies(response)
  if(cookie_lines = response.get_fields('set-cookie')) != nil
    cookie_lines.each do | cookie_line |
      cookies = cookie_line.split('; ').map { |v| v.split('=')}
      cookies.each { |c| @cookies[c[0]] = c[1] }
    end
  end
end