class Arachni::HTTP::Client
Provides a system-wide, simple and high-performance HTTP
client.
@author Tasos “Zapotek” Laskos <tasos.laskos@arachni-scanner.com>
Constants
Attributes
@return [Integer]
Amount of responses received for the running requests (of the current burst).
@return [Integer]
Sum of the response times for the running requests (of the current burst).
@return [Dynamic404Handler]
@return [Hash]
Default headers for {Request requests}.
@return [Integer]
Amount of performed requests.
@return [Integer]
Amount of received responses.
@return [Integer]
Amount of timed-out requests.
@return [String]
Framework target URL, used as reference.
Public Class Methods
# File lib/arachni/http/client.rb, line 501 def self.method_missing( sym, *args, &block ) instance.send( sym, *args, &block ) end
Arachni::Support::Mixins::Observable::new
# File lib/arachni/http/client.rb, line 123 def initialize super reset end
Private Class Methods
# File lib/arachni/http/client.rb, line 660 def self.info { name: 'HTTP' } end
Public Instance Methods
Aborts the running requests on a best effort basis.
# File lib/arachni/http/client.rb, line 278 def abort exception_jail { client_abort } end
@return [Float]
Average response time for the running requests (i.e. the current burst).
# File lib/arachni/http/client.rb, line 313 def burst_average_response_time return 0 if @burst_response_count == 0 @burst_response_time_sum / Float( @burst_response_count ) end
@return [Float]
Responses/second for the running requests (i.e. the current burst).
# File lib/arachni/http/client.rb, line 320 def burst_responses_per_second if @async_burst_response_count > 0 && burst_runtime > 0 return @async_burst_response_count / burst_runtime end 0 end
@return [Float]
Amount of time (in seconds) that the current burst has been running.
# File lib/arachni/http/client.rb, line 306 def burst_runtime @burst_runtime.to_i > 0 ? @burst_runtime : Time.now - (@burst_runtime_start || Time.now) end
Performs a ‘GET` {Request request} sending the headers in `:parameters`.
@param (see request
) @return (see request
)
@see request
# File lib/arachni/http/client.rb, line 469 def header( url = @url, options = {}, &block ) options[:headers] ||= {} options[:headers].merge!( (options.delete( :parameters ) || {}).dup ) request( url, options, &block ) end
# File lib/arachni/http/client.rb, line 505 def inspect s = "#<#{self.class} " statistics.each { |k, v| s << "@#{k}=#{v.inspect} " } s << '>' end
@return [Integer]
Current maximum concurrency of HTTP requests.
# File lib/arachni/http/client.rb, line 335 def max_concurrency @hydra.max_concurrency end
@param [Integer] concurrency
Sets the maximum concurrency of HTTP requests.
# File lib/arachni/http/client.rb, line 329 def max_concurrency=( concurrency ) @hydra.max_concurrency = concurrency end
@param [Request] request
Request to queue.
# File lib/arachni/http/client.rb, line 477 def queue( request ) notify_on_queue( request ) forward_request( request ) end
Queues/performs a generic request.
@param [String] url
URL to request.
@param [Hash] options
{Request#initialize Request options} with the following extras:
@option options [Hash] :cookies ({})
Extra cookies to use for this request.
@option options [Hash] :no_cookie_jar (false)
Do not include cookies from the {#cookie_jar}.
@param [Block] block Callback to be passed the {Response response}.
@return [Request, Response]
{Request} when operating in `:async:` `:mode` (the default), {Response} when in `:async:` `:mode`.
# File lib/arachni/http/client.rb, line 360 def request( url = @url, options = {}, &block ) fail ArgumentError, 'URL cannot be empty.' if !url options = options.dup cookies = options.delete( :cookies ) || {} raw_cookies = options.delete( :raw_cookies ) || [] raw_cookie_names = Set.new( raw_cookies.map(&:name) ) exception_jail false do if !options.delete( :no_cookie_jar ) raw_cookies |= begin cookie_jar.for_url( url ).reject do |c| cookies.include?( c.name ) || raw_cookie_names.include?( c.name ) end rescue => e print_error "Could not get cookies for URL '#{url}' from Cookiejar (#{e})." print_error_backtrace e [] end end on_headers = options.delete(:on_headers) on_body = options.delete(:on_body) on_body_line = options.delete(:on_body_line) on_body_lines = options.delete(:on_body_lines) request = Request.new( options.merge( url: url, headers: headers.merge( options.delete( :headers ) || {}, false ), cookies: cookies, raw_cookies: raw_cookies )) if on_headers request.on_headers( &on_headers ) end if on_body request.on_body( &on_body ) end if on_body_line request.on_body_line( &on_body_line ) end if on_body_lines request.on_body_lines( &on_body_lines ) end if block_given? request.on_complete( &block ) end queue( request ) return request.run if request.blocking? request end end
@return [Arachni::HTTP]
Reset `self`.
# File lib/arachni/http/client.rb, line 145 def reset( hooks_too = true ) clear_observers if hooks_too State.http.clear @url = Options.url.to_s @url = nil if @url.empty? client_initialize reset_options if Options.http.cookie_jar_filepath cookie_jar.load( Options.http.cookie_jar_filepath ) end Options.http.cookies.each do |name, value| update_cookies( name => value ) end if Options.http.cookie_string update_cookies( Options.http.cookie_string ) end reset_burst_info @request_count = 0 @async_response_count = 0 @response_count = 0 @time_out_count = 0 @total_response_time_sum = 0 @total_runtime = 0 @queue_size = 0 @dynamic_404_handler = Dynamic404Handler.new self end
# File lib/arachni/http/client.rb, line 128 def reset_options @original_max_concurrency = Options.http.request_concurrency || MAX_CONCURRENCY self.max_concurrency = @original_max_concurrency headers.clear headers.merge!( 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'User-Agent' => Options.http.user_agent, 'Accept-Language' => 'en-US,en;q=0.8,he;q=0.6', SEED_HEADER_NAME => Arachni::Utilities.random_seed ) headers['From'] = Options.authorized_by if Options.authorized_by headers.merge!( Options.http.request_headers, false ) end
Runs all queued requests
# File lib/arachni/http/client.rb, line 219 def run exception_jail false do @burst_runtime = nil begin run_and_update_statistics duped_after_run = observers_for( :after_run ).dup observers_for( :after_run ).clear duped_after_run.each { |block| block.call } end while @queue_size > 0 || observers_for( :after_run ).any? notify_after_each_run # Prune the custom 404 cache after callbacks have been called. @dynamic_404_handler.prune @curr_res_time = 0 @curr_res_cnt = 0 true end end
@note Cookies or new callbacks set as a result of the block won’t affect
the HTTP singleton.
@param [Block] block
Block to executes inside a sandbox.
@return [Object]
Return value of the block.
# File lib/arachni/http/client.rb, line 251 def sandbox( &block ) h = {} instance_variables.each do |iv| val = instance_variable_get( iv ) h[iv] = val.deep_clone rescue val.dup rescue val end saved_observers = dup_observers pre_cookies = cookies.deep_clone pre_headers = headers.deep_clone ret = block.call( self ) cookie_jar.clear update_cookies pre_cookies headers.clear headers.merge! pre_headers h.each { |iv, val| instance_variable_set( iv, val ) } set_observers( saved_observers ) ret end
@return [Hash]
Hash including HTTP client statistics including: * {#request_count} * {#response_count} * {#time_out_count} * {#total_responses_per_second} * {#total_average_response_time} * {#burst_response_time_sum} * {#burst_response_count} * {#burst_responses_per_second} * {#burst_average_response_time} * {#max_concurrency} * {#original_max_concurrency}
# File lib/arachni/http/client.rb, line 200 def statistics [:request_count, :response_count, :time_out_count, :total_responses_per_second, :burst_response_time_sum, :burst_response_count, :burst_responses_per_second, :burst_average_response_time, :total_average_response_time, :max_concurrency, :original_max_concurrency]. inject({}) { |h, k| h[k] = send(k); h } end
@return [Float]
Average response time for all requests.
# File lib/arachni/http/client.rb, line 291 def total_average_response_time return 0 if @response_count == 0 @total_response_time_sum / Float( @response_count ) end
@return [Float] Responses/second.
# File lib/arachni/http/client.rb, line 297 def total_responses_per_second if @async_response_count > 0 && total_runtime > 0 return @async_response_count / Float( total_runtime ) end 0 end
@return [Integer]
Amount of time (in seconds) that has been devoted to performing requests and getting responses.
# File lib/arachni/http/client.rb, line 285 def total_runtime @total_runtime > 0 ? @total_runtime : burst_runtime end
Private Instance Methods
# File lib/arachni/http/client.rb, line 642 def client_abort @hydra.abort end
# File lib/arachni/http/client.rb, line 631 def client_initialize @hydra = Typhoeus::Hydra.new end
# File lib/arachni/http/client.rb, line 646 def client_queue( request ) if request.high_priority? @hydra.queue_front( request.to_typhoeus ) else @hydra.queue( request.to_typhoeus ) end true end
# File lib/arachni/http/client.rb, line 635 def client_run # Can get Ethon select errors. exception_jail( false ) { @hydra.run } Arachni.collect_young_objects if @queue_size > 0 end
# File lib/arachni/http/client.rb, line 656 def emergency_run? @queue_size >= Options.http.request_queue_size && !@running end
Performs the actual queueing of requests, passes them to Hydra and sets up callbacks and hooks.
@param [Request] request
# File lib/arachni/http/client.rb, line 539 def forward_request( request ) add_callbacks = !request.id request.id = @request_count if debug_level_3? print_debug_level_4 '------------' print_debug_level_4 'Queued request.' print_debug_level_4 "ID#: #{request.id}" print_debug_level_4 "Performer: #{request.performer.inspect}" print_debug_level_4 "URL: #{request.url}" print_debug_level_4 "Method: #{request.method}" print_debug_level_4 "Params: #{request.parameters}" print_debug_level_4 "Body: #{request.body}" print_debug_level_4 "Headers: #{request.headers}" print_debug_level_4 "Cookies: #{request.cookies}" print_debug_level_4 "Train?: #{request.train?}" print_debug_level_4 "Fingerprint?: #{request.fingerprint?}" print_debug_level_4 '------------' end if add_callbacks @global_on_complete ||= method(:global_on_complete) request.on_complete( &@global_on_complete ) end synchronize { @request_count += 1 } return if request.blocking? if client_queue( request ) @queue_size += 1 if emergency_run? print_info 'Request queue reached its maximum size, performing an emergency run.' run_and_update_statistics end end request end
# File lib/arachni/http/client.rb, line 580 def global_on_complete( response ) request = response.request synchronize do @response_count += 1 @burst_response_count += 1 if request.asynchronous? @async_response_count += 1 @async_burst_response_count += 1 end response_time = response.timed_out? ? request.timeout / 1_000.0 : response.time @burst_response_time_sum += response_time @total_response_time_sum += response_time if response.request.fingerprint? && Platform::Manager.fingerprint?( response ) # Force a fingerprint by converting the Response to a Page object. response.to_page end notify_on_complete( response ) parse_and_set_cookies( response ) if request.update_cookies? if debug_level_3? print_debug_level_4 '------------' print_debug_level_4 "Got response for request ID#: #{response.request.id}\n#{response.request}" print_debug_level_4 "Performer: #{response.request.performer.inspect}" print_debug_level_4 "Status: #{response.code}" print_debug_level_4 "Code: #{response.return_code}" print_debug_level_4 "Message: #{response.return_message}" print_debug_level_4 "URL: #{response.url}" print_debug_level_4 "Headers:\n#{response.headers_string}" print_debug_level_4 "Parsed headers: #{response.headers}" end if response.timed_out? print_debug_level_4 "Request timed-out! -- ID# #{response.request.id}" @time_out_count += 1 end print_debug_level_4 '------------' end end
# File lib/arachni/http/client.rb, line 527 def reset_burst_info @burst_response_time_sum = 0 @burst_response_count = 0 @async_burst_response_count = 0 @burst_runtime = 0 @burst_runtime_start = Time.now end
# File lib/arachni/http/client.rb, line 513 def run_and_update_statistics @running = true reset_burst_info client_run @queue_size = 0 @running = false @burst_runtime += Time.now - @burst_runtime_start @total_runtime += @burst_runtime end