class PSGC::Import::DownloadManager
Responsible for fetching and caching pages
Constants
- CHECKSUMS
- MAX_TRIES
Public Class Methods
already_there(target, full_target)
click to toggle source
Check if file exists and its MD5 hash equals expected_md5, if present
# File lib/psgc/import.rb, line 1835 def already_there(target, full_target) File.exists?(full_target) and (!CHECKSUMS.key?(target) or Digest::MD5.file(full_target).hexdigest == CHECKSUMS[target]) end
cmd(cmd)
click to toggle source
Shortcut for:
puts(cmd); system(cmd)
# File lib/psgc/import.rb, line 1843 def cmd(cmd) puts cmd system cmd end
fetch(src, target)
click to toggle source
# File lib/psgc/import.rb, line 1811 def fetch(src, target) full_source = URI.join(Base.uri, src) full_target = File.join(Base.dir, target) if already_there(target, full_target) puts "#{target} already exists and matches expected hash, skipping" else puts "#{target} exists but doesn't match expected hash" if File.exists?(full_target) tries = MAX_TRIES success = false loop do max_time = 60 * (1 + (MAX_TRIES-tries)**2) success = cmd("curl -m #{max_time} \"#{full_source}\" > #{full_target}") return if success tries -= 1 break if tries == 0 end puts "`curl \"#{src}\" > #{target}` failed after multiple tries, giving up" exit 1 end end