class ProxyFetcher::HTTPClient

Default ProxyFetcher HTTP client used to fetch proxy lists from the different providers. Uses ProxyFetcher configuration options for sending HTTP requests to providers URLs.

Attributes

headers[R]

@!attribute [r] HTTP headers

@return [Hash] headers
http[R]

@!attribute [r] http

@return [Net::HTTP] HTTP client
method[R]

@!attribute [r] HTTP method

@return [String] HTTP method verb
params[R]

@!attribute [r] HTTP params

@return [Hash] params
ssl_ctx[R]

@!attribute [r] ssl_ctx

@return [OpenSSL::SSL::SSLContext] SSL context
timeout[R]

@!attribute [r] timeout

@return [Integer] Request timeout
url[R]

@!attribute [r] url

@return [String] URL

Public Class Methods

fetch(*args, **kwargs, &block) click to toggle source

Fetches resource content by sending HTTP request to it. Synthetic sugar to simplify URIes fetching.

@param url [String] URL

@return [String]

resource content
# File lib/proxy_fetcher/utils/http_client.rb, line 44
def self.fetch(*args, **kwargs, &block)
  new(*args, **kwargs, &block).fetch
end
new(url, method: :get, params: {}, headers: {}) click to toggle source

Initialize HTTP client instance

@return [HTTPClient]

# File lib/proxy_fetcher/utils/http_client.rb, line 52
def initialize(url, method: :get, params: {}, headers: {})
  @url = url.to_s
  @method = method.to_sym
  @params = params
  @headers = headers

  unless HTTP::Request::METHODS.include?(@method)
    raise ArgumentError, "'#{@method}' is a wrong HTTP method name"
  end

  @timeout = ProxyFetcher.config.provider_proxies_load_timeout
  @http = build_http_engine
  @ssl_ctx = build_ssl_context
end

Public Instance Methods

fetch(**options) click to toggle source

Fetches resource content by sending HTTP request to it.

@return [String]

response body
# File lib/proxy_fetcher/utils/http_client.rb, line 72
def fetch(**options)
  response = perform_http_request
  return response if options.fetch(:raw, false)

  response.body.to_s
rescue StandardError => e
  ProxyFetcher.config.logger.warn("Failed to process request to #{url} (#{e.message})")
  ""
end

Protected Instance Methods

build_http_engine() click to toggle source
# File lib/proxy_fetcher/utils/http_client.rb, line 90
def build_http_engine
  HTTP.headers(default_headers.merge(headers)).timeout(connect: timeout, read: timeout)
end
build_ssl_context() click to toggle source
# File lib/proxy_fetcher/utils/http_client.rb, line 84
def build_ssl_context
  OpenSSL::SSL::SSLContext.new.tap do |context|
    context.verify_mode = OpenSSL::SSL::VERIFY_NONE
  end
end
default_headers() click to toggle source

Default HTTP client headers

@return [Hash]

hash of HTTP headers
# File lib/proxy_fetcher/utils/http_client.rb, line 108
def default_headers
  {
    "User-Agent" => ProxyFetcher.config.user_agent
  }
end
perform_http_request(http_method: method, http_params: params) click to toggle source
# File lib/proxy_fetcher/utils/http_client.rb, line 94
def perform_http_request(http_method: method, http_params: params)
  http.public_send(
    http_method,
    url,
    form: http_params,
    ssl_context: ssl_ctx
  )
end