class ProxyFetcher::HTTPClient
Default ProxyFetcher
HTTP client used to fetch proxy lists from the different providers. Uses ProxyFetcher
configuration options for sending HTTP requests to providers URLs.
Attributes
headers[R]
@!attribute [r] HTTP headers
@return [Hash] headers
http[R]
@!attribute [r] http
@return [Net::HTTP] HTTP client
method[R]
@!attribute [r] HTTP method
@return [String] HTTP method verb
params[R]
@!attribute [r] HTTP params
@return [Hash] params
ssl_ctx[R]
@!attribute [r] ssl_ctx
@return [OpenSSL::SSL::SSLContext] SSL context
timeout[R]
@!attribute [r] timeout
@return [Integer] Request timeout
url[R]
@!attribute [r] url
@return [String] URL
Public Class Methods
fetch(*args, **kwargs, &block)
click to toggle source
Fetches resource content by sending HTTP request to it. Synthetic sugar to simplify URIes fetching.
@param url [String] URL
@return [String]
resource content
# File lib/proxy_fetcher/utils/http_client.rb, line 44 def self.fetch(*args, **kwargs, &block) new(*args, **kwargs, &block).fetch end
new(url, method: :get, params: {}, headers: {})
click to toggle source
Initialize HTTP client instance
@return [HTTPClient]
# File lib/proxy_fetcher/utils/http_client.rb, line 52 def initialize(url, method: :get, params: {}, headers: {}) @url = url.to_s @method = method.to_sym @params = params @headers = headers unless HTTP::Request::METHODS.include?(@method) raise ArgumentError, "'#{@method}' is a wrong HTTP method name" end @timeout = ProxyFetcher.config.provider_proxies_load_timeout @http = build_http_engine @ssl_ctx = build_ssl_context end
Public Instance Methods
fetch(**options)
click to toggle source
Fetches resource content by sending HTTP request to it.
@return [String]
response body
# File lib/proxy_fetcher/utils/http_client.rb, line 72 def fetch(**options) response = perform_http_request return response if options.fetch(:raw, false) response.body.to_s rescue StandardError => e ProxyFetcher.config.logger.warn("Failed to process request to #{url} (#{e.message})") "" end
Protected Instance Methods
build_http_engine()
click to toggle source
# File lib/proxy_fetcher/utils/http_client.rb, line 90 def build_http_engine HTTP.headers(default_headers.merge(headers)).timeout(connect: timeout, read: timeout) end
build_ssl_context()
click to toggle source
# File lib/proxy_fetcher/utils/http_client.rb, line 84 def build_ssl_context OpenSSL::SSL::SSLContext.new.tap do |context| context.verify_mode = OpenSSL::SSL::VERIFY_NONE end end
default_headers()
click to toggle source
Default HTTP client headers
@return [Hash]
hash of HTTP headers
# File lib/proxy_fetcher/utils/http_client.rb, line 108 def default_headers { "User-Agent" => ProxyFetcher.config.user_agent } end
perform_http_request(http_method: method, http_params: params)
click to toggle source
# File lib/proxy_fetcher/utils/http_client.rb, line 94 def perform_http_request(http_method: method, http_params: params) http.public_send( http_method, url, form: http_params, ssl_context: ssl_ctx ) end