class PSGC::Import::DownloadManager

Responsible for fetching and caching pages

Constants

CHECKSUMS
MAX_TRIES

Public Class Methods

already_there(target, full_target) click to toggle source

Check if file exists and its MD5 hash equals expected_md5, if present

# File lib/psgc/import.rb, line 1835
def already_there(target, full_target)
  File.exists?(full_target) and (!CHECKSUMS.key?(target) or Digest::MD5.file(full_target).hexdigest == CHECKSUMS[target])
end
cmd(cmd) click to toggle source

Shortcut for:

puts(cmd); system(cmd)
# File lib/psgc/import.rb, line 1843
def cmd(cmd)
  puts cmd
  system cmd
end
fetch(src, target) click to toggle source
# File lib/psgc/import.rb, line 1811
def fetch(src, target)
  full_source = URI.join(Base.uri, src)
  full_target = File.join(Base.dir, target)
  if already_there(target, full_target)
    puts "#{target} already exists and matches expected hash, skipping"
  else
    puts "#{target} exists but doesn't match expected hash" if File.exists?(full_target)
    tries = MAX_TRIES
    success = false

    loop do
      max_time = 60 * (1 + (MAX_TRIES-tries)**2)
      success = cmd("curl -m #{max_time} \"#{full_source}\" > #{full_target}")
      return if success
      tries -= 1
      break if tries == 0
    end

    puts "`curl \"#{src}\" > #{target}` failed after multiple tries, giving up"
    exit 1
  end
end