class ProxyFetcher::Manager
ProxyFetcher
Manager
class for interacting with proxy lists from various providers.
Constants
- REFRESHER_LOCK
Attributes
@!attribute [r] proxies
@return [Array<ProxyFetcher::Proxy>] An array of proxies
Public Class Methods
# File lib/proxy_fetcher/manager.rb, line 9 def from_files(files, **options) new(**options.merge(files: Array(files))) end
Initialize ProxyFetcher
Manager
instance for managing proxies
refresh: true - load proxy list from the remote server on initialization refresh: false - just initialize the class, proxy list will be empty ([])
@return [Manager]
# File lib/proxy_fetcher/manager.rb, line 27 def initialize(**options) if options.fetch(:refresh, true) refresh_list!(options.fetch(:filters, {})) else @proxies = [] end files = Array(options.fetch(:file, options.fetch(:files, []))) load_proxies_from_files!(files) if files&.any? cleanup! if options.fetch(:validate, false) end
Public Instance Methods
Clean current proxy list from dead proxies (that doesn't respond by timeout)
@return [Array<ProxyFetcher::Proxy>]
list of valid proxies
# File lib/proxy_fetcher/manager.rb, line 129 def cleanup! valid_proxies = ProxyListValidator.new(@proxies).validate @proxies &= valid_proxies end
Pop just first proxy (and back it to the end of the proxy list).
@return [ProxyFetcher::Proxy, NilClass]
proxy object from the list
# File lib/proxy_fetcher/manager.rb, line 74 def get return if @proxies.empty? first_proxy = @proxies.shift @proxies << first_proxy first_proxy end
Pop first valid proxy (and back it to the end of the proxy list) Invalid proxies will be removed from the list
@return [ProxyFetcher::Proxy, NilClass]
proxy object from the list
# File lib/proxy_fetcher/manager.rb, line 91 def get! index = proxies.find_index(&:connectable?) return if index.nil? proxy = proxies.delete_at(index) tail = proxies[index..-1] @proxies = tail << proxy proxy end
@private No need to put all the attr_readers to the output
# File lib/proxy_fetcher/manager.rb, line 157 def inspect to_s end
Loads proxies from files.
@param proxy_files [String, Array<String,Pathname>]
file path of list of files to load
# File lib/proxy_fetcher/manager.rb, line 110 def load_proxies_from_files!(proxy_files) proxy_files = Array(proxy_files) return if proxy_files.empty? proxy_files.each do |proxy_file| File.foreach(proxy_file, chomp: true) do |proxy_string| addr, port = proxy_string.split(":", 2) port = Integer(port) if port @proxies << Proxy.new(addr: addr, port: port) end end @proxies.uniq! end
Returns random proxy
@return [Proxy]
random proxy from the loaded list
# File lib/proxy_fetcher/manager.rb, line 141 def random_proxy proxies.sample end
Returns array of proxy URLs (just schema + host + port)
@return [Array<String>]
collection of proxies
# File lib/proxy_fetcher/manager.rb, line 152 def raw_proxies proxies.map(&:url) end
Update current proxy list using configured providers.
@param filters [Hash] providers filters
# File lib/proxy_fetcher/manager.rb, line 44 def refresh_list!(filters = nil) @proxies = [] threads = [] ProxyFetcher.config.providers.each do |provider_name| threads << Thread.new do Thread.current.report_on_exception = false provider = ProxyFetcher::Configuration.providers_registry.class_for(provider_name) provider_filters = filters && filters.fetch(provider_name.to_sym, filters) provider_proxies = provider.fetch_proxies!(provider_filters) REFRESHER_LOCK.synchronize do @proxies.concat(provider_proxies) end end end threads.each(&:join) @proxies end