class Goodcheck::ImportLoader
Attributes
cache_path[R]
config_path[R]
expires_in[R]
force_download[R]
Public Class Methods
new(cache_path:, expires_in: 3 * 60, force_download:, config_path:)
click to toggle source
# File lib/goodcheck/import_loader.rb, line 35 def initialize(cache_path:, expires_in: 3 * 60, force_download:, config_path:) @cache_path = cache_path @expires_in = expires_in @force_download = force_download @config_path = config_path end
Public Instance Methods
cache_name(uri)
click to toggle source
# File lib/goodcheck/import_loader.rb, line 79 def cache_name(uri) Digest::SHA2.hexdigest(uri.to_s) end
http_get(uri, limit = 10)
click to toggle source
# File lib/goodcheck/import_loader.rb, line 131 def http_get(uri, limit = 10) raise ArgumentError, "Too many HTTP redirects" if limit == 0 max_retry_count = 2 retry_count = 0 begin res = Net::HTTP.get_response URI(uri) case res when Net::HTTPSuccess res.body when Net::HTTPRedirection location = res['Location'] http_get location, limit - 1 when Net::HTTPClientError, Net::HTTPServerError raise HTTPGetError.new(res) else raise Error, "HTTP GET failed due to #{res.inspect}" end rescue Net::OpenTimeout, HTTPGetError => exn if retry_count < max_retry_count retry_count += 1 Goodcheck.logger.info "Retry ##{retry_count} - HTTP GET #{uri} due to #{exn.inspect}..." sleep 1 retry else raise end end end
load(name, &block)
click to toggle source
# File lib/goodcheck/import_loader.rb, line 42 def load(name, &block) uri = begin URI.parse(name) rescue URI::InvalidURIError nil end case uri&.scheme when nil load_file name, &block when "file" load_file uri.path, &block when "http", "https" load_http uri, &block else raise UnexpectedSchemaError.new(uri) end end
load_file(path, &block)
click to toggle source
# File lib/goodcheck/import_loader.rb, line 61 def load_file(path, &block) files = Pathname.glob(File.join(config_path.parent.to_path, path), File::FNM_DOTMATCH | File::FNM_EXTGLOB).sort if files.empty? raise FileNotFound.new(path) else files.each do |file| Goodcheck.logger.info "Reading file: #{file}" if unarchiver.tar_gz?(file) unarchiver.tar_gz(file.read) do |content, filename| block.call(content, filename) end else block.call(file.read, file.to_path) end end end end
load_http(uri, &block)
click to toggle source
# File lib/goodcheck/import_loader.rb, line 83 def load_http(uri, &block) hash = cache_name(uri) path = cache_path + hash Goodcheck.logger.info "Calculated cache name: #{hash}" download = false if force_download Goodcheck.logger.debug "Downloading: force flag" download = true end if !download && !path.file? Goodcheck.logger.debug "Downloading: no cache found" download = true end if !download && path.mtime + expires_in < Time.now Goodcheck.logger.debug "Downloading: cache expired" download = true end if download path.rmtree if path.exist? Goodcheck.logger.info "Downloading content..." if unarchiver.tar_gz?(uri.path) unarchiver.tar_gz(http_get(uri)) do |content, filename| block.call(content, filename) write_cache "#{uri}/#{filename}", content end else content = http_get(uri) block.call(content, uri.path) write_cache uri, content end else Goodcheck.logger.info "Reading content from cache..." block.call(path.read, path.to_path) end end
write_cache(uri, content)
click to toggle source
# File lib/goodcheck/import_loader.rb, line 125 def write_cache(uri, content) path = cache_path + cache_name(uri) path.write(content) end
Private Instance Methods
unarchiver()
click to toggle source
# File lib/goodcheck/import_loader.rb, line 163 def unarchiver @unarchiver ||= begin filter = ->(filename) { %w[.yml .yaml].include?(File.extname(filename).downcase) && File.basename(filename) != DEFAULT_CONFIG_FILE } Unarchiver.new(file_filter: filter) end end