class Arachni::HTTP::Client

Provides a system-wide, simple and high-performance HTTP client.

@author Tasos “Zapotek” Laskos <tasos.laskos@arachni-scanner.com>

Constants

HTTP_TIMEOUT

Default 1 minute timeout for HTTP requests.

MAX_CONCURRENCY

Default maximum concurrency for HTTP requests.

SEED_HEADER_NAME

Attributes

burst_response_count[R]

@return [Integer]

Amount of responses received for the running requests (of the current burst).
burst_response_time_sum[R]

@return [Integer]

Sum of the response times for the running requests (of the current burst).
dynamic_404_handler[R]

@return [Dynamic404Handler]

headers[R]

@return [Hash]

Default headers for {Request requests}.
original_max_concurrency[R]
request_count[R]

@return [Integer]

Amount of performed requests.
response_count[R]

@return [Integer]

Amount of received responses.
time_out_count[R]

@return [Integer]

Amount of timed-out requests.
url[R]

@return [String]

Framework target URL, used as reference.

Public Class Methods

method_missing( sym, *args, &block ) click to toggle source
# File lib/arachni/http/client.rb, line 501
def self.method_missing( sym, *args, &block )
    instance.send( sym, *args, &block )
end
new() click to toggle source
# File lib/arachni/http/client.rb, line 123
def initialize
    super
    reset
end

Private Class Methods

info() click to toggle source
# File lib/arachni/http/client.rb, line 660
def self.info
    { name: 'HTTP' }
end

Public Instance Methods

abort() click to toggle source

Aborts the running requests on a best effort basis.

# File lib/arachni/http/client.rb, line 278
def abort
    exception_jail { client_abort }
end
burst_average_response_time() click to toggle source

@return [Float]

Average response time for the running requests (i.e. the current burst).
# File lib/arachni/http/client.rb, line 313
def burst_average_response_time
    return 0 if @burst_response_count == 0
    @burst_response_time_sum / Float( @burst_response_count )
end
burst_responses_per_second() click to toggle source

@return [Float]

Responses/second for the running requests (i.e. the current burst).
# File lib/arachni/http/client.rb, line 320
def burst_responses_per_second
    if @async_burst_response_count > 0 && burst_runtime > 0
        return @async_burst_response_count / burst_runtime
    end
    0
end
burst_runtime() click to toggle source

@return [Float]

Amount of time (in seconds) that the current burst has been running.
# File lib/arachni/http/client.rb, line 306
def burst_runtime
    @burst_runtime.to_i > 0 ?
        @burst_runtime : Time.now - (@burst_runtime_start || Time.now)
end
cookies() click to toggle source

@return [Array<Arachni::Element::Cookie>]

All cookies in the jar.
# File lib/arachni/http/client.rb, line 341
def cookies
    cookie_jar.cookies
end
get( url = @url, options = {}, &block ) click to toggle source

Performs a ‘GET` {Request request}.

@param (see request) @return (see request)

@see request

# File lib/arachni/http/client.rb, line 426
def get( url = @url, options = {}, &block )
    request( url, options, &block )
end
header( url = @url, options = {}, &block ) click to toggle source

Performs a ‘GET` {Request request} sending the headers in `:parameters`.

@param (see request) @return (see request)

@see request

# File lib/arachni/http/client.rb, line 469
def header( url = @url, options = {}, &block )
    options[:headers] ||= {}
    options[:headers].merge!( (options.delete( :parameters ) || {}).dup )
    request( url, options, &block )
end
inspect() click to toggle source
# File lib/arachni/http/client.rb, line 505
def inspect
    s = "#<#{self.class} "
    statistics.each { |k, v| s << "@#{k}=#{v.inspect} " }
    s << '>'
end
max_concurrency() click to toggle source

@return [Integer]

Current maximum concurrency of HTTP requests.
# File lib/arachni/http/client.rb, line 335
def max_concurrency
    @hydra.max_concurrency
end
max_concurrency=( concurrency ) click to toggle source

@param [Integer] concurrency

Sets the maximum concurrency of HTTP requests.
# File lib/arachni/http/client.rb, line 329
def max_concurrency=( concurrency )
    @hydra.max_concurrency = concurrency
end
parse_and_set_cookies( response ) click to toggle source

@note Runs {#on_new_cookies} callbacks.

@param [Response] response

Extracts cookies from `response` and updates the cookie-jar.
# File lib/arachni/http/client.rb, line 494
def parse_and_set_cookies( response )
    cookies = Cookie.from_response( response )
    update_cookies( cookies )

    notify_on_new_cookies( cookies, response )
end
post( url = @url, options = {}, &block ) click to toggle source

Performs a ‘POST` {Request request}.

@param (see request) @return (see request)

@see request

# File lib/arachni/http/client.rb, line 436
def post( url = @url, options = {}, &block )
    options[:body] = (options.delete( :parameters ) || {}).dup
    request( url, options.merge( method: :post ), &block )
end
queue( request ) click to toggle source

@param [Request] request

Request to queue.
# File lib/arachni/http/client.rb, line 477
def queue( request )
    notify_on_queue( request )
    forward_request( request )
end
request( url = @url, options = {}, &block ) click to toggle source

Queues/performs a generic request.

@param [String] url

URL to request.

@param [Hash] options

{Request#initialize Request options} with the following extras:

@option options [Hash] :cookies ({})

Extra cookies to use for this request.

@option options [Hash] :no_cookie_jar (false)

Do not include cookies from the {#cookie_jar}.

@param [Block] block Callback to be passed the {Response response}.

@return [Request, Response]

{Request} when operating in `:async:` `:mode` (the default), {Response}
when in `:async:` `:mode`.
# File lib/arachni/http/client.rb, line 360
def request( url = @url, options = {}, &block )
    fail ArgumentError, 'URL cannot be empty.' if !url

    options     = options.dup
    cookies     = options.delete( :cookies )     || {}
    raw_cookies = options.delete( :raw_cookies ) || []
    raw_cookie_names = Set.new( raw_cookies.map(&:name) )

    exception_jail false do
        if !options.delete( :no_cookie_jar )
            raw_cookies |= begin
                cookie_jar.for_url( url ).reject do |c|
                    cookies.include?( c.name ) ||
                        raw_cookie_names.include?( c.name )
                end
            rescue => e
                print_error "Could not get cookies for URL '#{url}' from Cookiejar (#{e})."
                print_error_backtrace e
                []
            end
        end

        on_headers    = options.delete(:on_headers)
        on_body       = options.delete(:on_body)
        on_body_line  = options.delete(:on_body_line)
        on_body_lines = options.delete(:on_body_lines)

        request = Request.new( options.merge(
            url:         url,
            headers:     headers.merge( options.delete( :headers ) || {}, false ),
            cookies:     cookies,
            raw_cookies: raw_cookies
        ))

        if on_headers
            request.on_headers( &on_headers )
        end

        if on_body
            request.on_body( &on_body )
        end

        if on_body_line
            request.on_body_line( &on_body_line )
        end

        if on_body_lines
            request.on_body_lines( &on_body_lines )
        end

        if block_given?
            request.on_complete( &block )
        end

        queue( request )
        return request.run if request.blocking?
        request
    end
end
reset( hooks_too = true ) click to toggle source

@return [Arachni::HTTP]

Reset `self`.
# File lib/arachni/http/client.rb, line 145
def reset( hooks_too = true )
    clear_observers if hooks_too
    State.http.clear

    @url = Options.url.to_s
    @url = nil if @url.empty?

    client_initialize

    reset_options

    if Options.http.cookie_jar_filepath
        cookie_jar.load( Options.http.cookie_jar_filepath )
    end

    Options.http.cookies.each do |name, value|
        update_cookies( name => value )
    end

    if Options.http.cookie_string
        update_cookies( Options.http.cookie_string )
    end

    reset_burst_info

    @request_count  = 0
    @async_response_count = 0
    @response_count = 0
    @time_out_count = 0

    @total_response_time_sum = 0
    @total_runtime           = 0

    @queue_size = 0

    @dynamic_404_handler = Dynamic404Handler.new

    self
end
reset_options() click to toggle source
# File lib/arachni/http/client.rb, line 128
def reset_options
    @original_max_concurrency = Options.http.request_concurrency || MAX_CONCURRENCY
    self.max_concurrency      = @original_max_concurrency

    headers.clear
    headers.merge!(
        'Accept'              => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'User-Agent'          => Options.http.user_agent,
        'Accept-Language'     => 'en-US,en;q=0.8,he;q=0.6',
        SEED_HEADER_NAME      => Arachni::Utilities.random_seed
    )
    headers['From'] = Options.authorized_by if Options.authorized_by
    headers.merge!( Options.http.request_headers, false )
end
run() click to toggle source

Runs all queued requests

# File lib/arachni/http/client.rb, line 219
def run
    exception_jail false do
        @burst_runtime = nil

        begin
            run_and_update_statistics

            duped_after_run = observers_for( :after_run ).dup
            observers_for( :after_run ).clear
            duped_after_run.each { |block| block.call }
        end while @queue_size > 0 || observers_for( :after_run ).any?

        notify_after_each_run

        # Prune the custom 404 cache after callbacks have been called.
        @dynamic_404_handler.prune

        @curr_res_time = 0
        @curr_res_cnt  = 0

        true
    end
end
sandbox( &block ) click to toggle source

@note Cookies or new callbacks set as a result of the block won’t affect

the HTTP singleton.

@param [Block] block

Block to executes  inside a sandbox.

@return [Object]

Return value of the block.
# File lib/arachni/http/client.rb, line 251
def sandbox( &block )
    h = {}
    instance_variables.each do |iv|
        val = instance_variable_get( iv )
        h[iv] = val.deep_clone rescue val.dup rescue val
    end

    saved_observers = dup_observers

    pre_cookies = cookies.deep_clone
    pre_headers = headers.deep_clone

    ret = block.call( self )

    cookie_jar.clear
    update_cookies pre_cookies

    headers.clear
    headers.merge! pre_headers

    h.each { |iv, val| instance_variable_set( iv, val ) }
    set_observers( saved_observers )

    ret
end
set_cookies( cookies )
Alias for: update_cookies
statistics() click to toggle source

@return [Hash]

Hash including HTTP client statistics including:

*  {#request_count}
*  {#response_count}
*  {#time_out_count}
*  {#total_responses_per_second}
*  {#total_average_response_time}
*  {#burst_response_time_sum}
*  {#burst_response_count}
*  {#burst_responses_per_second}
*  {#burst_average_response_time}
*  {#max_concurrency}
*  {#original_max_concurrency}
# File lib/arachni/http/client.rb, line 200
def statistics
   [:request_count, :response_count, :time_out_count,
    :total_responses_per_second, :burst_response_time_sum,
    :burst_response_count, :burst_responses_per_second,
    :burst_average_response_time, :total_average_response_time,
    :max_concurrency, :original_max_concurrency].
       inject({}) { |h, k| h[k] = send(k); h }
end
total_average_response_time() click to toggle source

@return [Float]

Average response time for all requests.
# File lib/arachni/http/client.rb, line 291
def total_average_response_time
    return 0 if @response_count == 0
    @total_response_time_sum / Float( @response_count )
end
total_responses_per_second() click to toggle source

@return [Float] Responses/second.

# File lib/arachni/http/client.rb, line 297
def total_responses_per_second
    if @async_response_count > 0 && total_runtime > 0
        return @async_response_count / Float( total_runtime )
    end
    0
end
total_runtime() click to toggle source

@return [Integer]

Amount of time (in seconds) that has been devoted to performing requests
and getting responses.
# File lib/arachni/http/client.rb, line 285
def total_runtime
    @total_runtime > 0 ? @total_runtime : burst_runtime
end
trace( url = @url, options = {}, &block ) click to toggle source

Performs a ‘TRACE` {Request request}.

@param (see request) @return (see request)

@see request

# File lib/arachni/http/client.rb, line 447
def trace( url = @url, options = {}, &block )
    request( url, options.merge( method: :trace ), &block )
end
update_cookies( cookies ) click to toggle source

@param [Array<String, Hash, Arachni::Element::Cookie>] cookies

Updates the cookie-jar with the passed `cookies`.
# File lib/arachni/http/client.rb, line 484
def update_cookies( cookies )
    cookie_jar.update( cookies )
    cookie_jar.cookies
end
Also aliased as: set_cookies

Private Instance Methods

client_abort() click to toggle source
# File lib/arachni/http/client.rb, line 642
def client_abort
    @hydra.abort
end
client_initialize() click to toggle source
# File lib/arachni/http/client.rb, line 631
def client_initialize
    @hydra = Typhoeus::Hydra.new
end
client_queue( request ) click to toggle source
# File lib/arachni/http/client.rb, line 646
def client_queue( request )
    if request.high_priority?
        @hydra.queue_front( request.to_typhoeus )
    else
        @hydra.queue( request.to_typhoeus )
    end

    true
end
client_run() click to toggle source
# File lib/arachni/http/client.rb, line 635
def client_run
    # Can get Ethon select errors.
    exception_jail( false ) { @hydra.run }

    Arachni.collect_young_objects if @queue_size > 0
end
emergency_run?() click to toggle source
# File lib/arachni/http/client.rb, line 656
def emergency_run?
    @queue_size >= Options.http.request_queue_size && !@running
end
forward_request( request ) click to toggle source

Performs the actual queueing of requests, passes them to Hydra and sets up callbacks and hooks.

@param [Request] request

# File lib/arachni/http/client.rb, line 539
def forward_request( request )
    add_callbacks = !request.id
    request.id    = @request_count

    if debug_level_3?
        print_debug_level_4 '------------'
        print_debug_level_4 'Queued request.'
        print_debug_level_4 "ID#: #{request.id}"
        print_debug_level_4 "Performer: #{request.performer.inspect}"
        print_debug_level_4 "URL: #{request.url}"
        print_debug_level_4 "Method: #{request.method}"
        print_debug_level_4 "Params: #{request.parameters}"
        print_debug_level_4 "Body: #{request.body}"
        print_debug_level_4 "Headers: #{request.headers}"
        print_debug_level_4 "Cookies: #{request.cookies}"
        print_debug_level_4 "Train?: #{request.train?}"
        print_debug_level_4 "Fingerprint?: #{request.fingerprint?}"
        print_debug_level_4  '------------'
    end

    if add_callbacks
        @global_on_complete ||= method(:global_on_complete)
        request.on_complete( &@global_on_complete )
    end

    synchronize { @request_count += 1 }

    return if request.blocking?

    if client_queue( request )
        @queue_size += 1

        if emergency_run?
            print_info 'Request queue reached its maximum size, performing an emergency run.'
            run_and_update_statistics
        end
    end

    request
end
global_on_complete( response ) click to toggle source
# File lib/arachni/http/client.rb, line 580
def global_on_complete( response )
    request = response.request

    synchronize do
        @response_count       += 1
        @burst_response_count += 1

        if request.asynchronous?
            @async_response_count       += 1
            @async_burst_response_count += 1
        end

        response_time = response.timed_out? ?
            request.timeout / 1_000.0 :
            response.time

        @burst_response_time_sum += response_time
        @total_response_time_sum += response_time

        if response.request.fingerprint? &&
            Platform::Manager.fingerprint?( response )

            # Force a fingerprint by converting the Response to a Page object.
            response.to_page
        end

        notify_on_complete( response )

        parse_and_set_cookies( response ) if request.update_cookies?

        if debug_level_3?
            print_debug_level_4 '------------'
            print_debug_level_4 "Got response for request ID#: #{response.request.id}\n#{response.request}"
            print_debug_level_4 "Performer: #{response.request.performer.inspect}"
            print_debug_level_4 "Status: #{response.code}"
            print_debug_level_4 "Code: #{response.return_code}"
            print_debug_level_4 "Message: #{response.return_message}"
            print_debug_level_4 "URL: #{response.url}"
            print_debug_level_4 "Headers:\n#{response.headers_string}"
            print_debug_level_4 "Parsed headers: #{response.headers}"
        end

        if response.timed_out?
            print_debug_level_4 "Request timed-out! -- ID# #{response.request.id}"
            @time_out_count += 1
        end

        print_debug_level_4 '------------'
    end
end
reset_burst_info() click to toggle source
# File lib/arachni/http/client.rb, line 527
def reset_burst_info
    @burst_response_time_sum = 0
    @burst_response_count    = 0
    @async_burst_response_count = 0
    @burst_runtime           = 0
    @burst_runtime_start     = Time.now
end
run_and_update_statistics() click to toggle source
# File lib/arachni/http/client.rb, line 513
def run_and_update_statistics
    @running = true

    reset_burst_info

    client_run

    @queue_size = 0
    @running    = false

    @burst_runtime += Time.now - @burst_runtime_start
    @total_runtime += @burst_runtime
end