class PushmiPullyu::AIP::Downloader

Download all of the metadata/datastreams and associated data related to an object

Constants

PREDICATE_URIS

Public Class Methods

new(entity, aip_directory) click to toggle source
# File lib/pushmi_pullyu/aip/downloader.rb, line 23
def initialize(entity, aip_directory)
  @entity = entity
  @entity_identifier = "[#{entity[:type]} - #{entity[:uuid]}]".freeze
  @aip_directory = aip_directory
end

Public Instance Methods

run() click to toggle source
# File lib/pushmi_pullyu/aip/downloader.rb, line 29
def run
  PushmiPullyu.logger.info("#{@entity_identifier}: Retreiving data from Jupiter ...")

  authenticate_http_calls
  make_directories

  # Main object metadata
  download_and_log(object_aip_paths[:main_object_remote],
                   object_aip_paths[:main_object_local])
  download_and_log(object_aip_paths[:file_sets_remote],
                   object_aip_paths[:file_sets_local])

  # Get file paths for processing
  file_paths = get_file_paths(object_aip_paths[:file_paths_remote])

  file_paths[:files].each do |file_path|
    file_uuid = file_path[:file_uuid]
    make_file_set_directories(file_uuid)
    copy_and_log(file_uuid, file_path)
    file_aip_path = file_aip_paths(file_uuid)
    download_and_log(file_aip_path[:fixity_remote],
                     file_aip_path[:fixity_local])
    download_and_log(file_aip_path[:file_set_remote],
                     file_aip_path[:file_set_local])
  end
end

Private Instance Methods

aip_dirs() click to toggle source

Directories

# File lib/pushmi_pullyu/aip/downloader.rb, line 145
def aip_dirs
  @aip_dirs ||= {
    objects: "#{@aip_directory}/data/objects",
    metadata: "#{@aip_directory}/data/objects/metadata",
    files: "#{@aip_directory}/data/objects/files",
    files_metadata: "#{@aip_directory}/data/objects/metadata/files_metadata",
    logs: "#{@aip_directory}/data/logs",
    file_logs: "#{@aip_directory}/data/logs/files_logs"
  }
end
authenticate_http_calls() click to toggle source
# File lib/pushmi_pullyu/aip/downloader.rb, line 80
def authenticate_http_calls
  @uri = URI.parse(PushmiPullyu.options[:jupiter][:jupiter_url])
  @http = Net::HTTP.new(@uri.host, @uri.port)
  request = Net::HTTP::Post.new(@uri.request_uri + 'auth/system')
  request.set_form_data(
    email: PushmiPullyu.options[:jupiter][:user],
    api_key: PushmiPullyu.options[:jupiter][:api_key]
  )
  response = @http.request(request)
  # If we cannot find the set-cookie header then the session was not set
  raise JupiterAuthenticationError if response.response['set-cookie'].nil?

  @cookies = response.response['set-cookie']
end
clean_directories() click to toggle source
# File lib/pushmi_pullyu/aip/downloader.rb, line 182
def clean_directories
  return unless File.exist?(@aip_directory)

  PushmiPullyu.logger.debug("#{@entity_identifier}: Nuking directories ...")
  FileUtils.rm_rf(@aip_directory)
end
compare_md5(local, remote_checksum) click to toggle source
# File lib/pushmi_pullyu/aip/downloader.rb, line 75
def compare_md5(local, remote_checksum)
  local_md5 = Digest::MD5.file local
  local_md5.base64digest == remote_checksum
end
copy_and_log(file_uuid, file_path) click to toggle source
# File lib/pushmi_pullyu/aip/downloader.rb, line 58
def copy_and_log(file_uuid, file_path)
  remote = file_path[:file_path]
  remote_checksum = file_path[:file_checksum]
  files_path = file_set_dirs(file_uuid)[:files]
  output_file = "#{files_path}/#{file_path[:file_name]}"
  log_downloading(remote, output_file)
  FileUtils.copy_file(remote, output_file)

  is_success = File.exist?(output_file) &&
               File.size(remote) == File.size(output_file) &&
               compare_md5(output_file, remote_checksum)

  log_saved(is_success, output_file)

  raise JupiterCopyError unless is_success
end
download_and_log(remote, local) click to toggle source
# File lib/pushmi_pullyu/aip/downloader.rb, line 95
def download_and_log(remote, local)
  log_downloading(remote, local)

  @uri = URI.parse(PushmiPullyu.options[:jupiter][:jupiter_url])
  request = Net::HTTP::Get.new(@uri.request_uri + remote)
  # add previously stored cookies
  request['Cookie'] = @cookies

  response = @http.request(request)
  is_success = if response.is_a?(Net::HTTPSuccess)
                 File.open(local, 'wb') do |file|
                   file.write(response.body)
                 end
                 # Response was a success and the file was saved to local
                 File.exist? local
               end

  log_saved(is_success, local)
  raise JupiterDownloadError unless is_success
end
file_aip_paths(file_set_uuid) click to toggle source
# File lib/pushmi_pullyu/aip/downloader.rb, line 203
def file_aip_paths(file_set_uuid)
  file_set_paths = file_set_dirs(file_set_uuid)
  @file_aip_paths ||= {}
  @file_aip_paths[file_set_uuid] ||= {
    fixity_remote: "#{object_uri}/filesets/#{file_set_uuid}/fixity",
    fixity_local: "#{file_set_paths[:logs]}/content_fixity_report.n3",
    file_set_remote: "#{object_uri}/filesets/#{file_set_uuid}",
    file_set_local: "#{file_set_paths[:metadata]}/file_set_metadata.n3"
  }.freeze
end
file_set_dirs(file_set_uuid) click to toggle source
# File lib/pushmi_pullyu/aip/downloader.rb, line 156
def file_set_dirs(file_set_uuid)
  @file_set_dirs ||= {}
  @file_set_dirs[file_set_uuid] ||= {
    metadata: "#{aip_dirs[:files_metadata]}/#{file_set_uuid}",
    files: "#{aip_dirs[:files]}/#{file_set_uuid}",
    logs: "#{aip_dirs[:file_logs]}/#{file_set_uuid}"
  }
end
get_file_paths(url) click to toggle source
# File lib/pushmi_pullyu/aip/downloader.rb, line 116
def get_file_paths(url)
  request = Net::HTTP::Get.new(@uri.request_uri + url)
  # add previously stored cookies
  request['Cookie'] = @cookies

  response = @http.request(request)

  JSON.parse(response.body, symbolize_names: true)
end
log_downloading(url, output_file) click to toggle source

Logging

# File lib/pushmi_pullyu/aip/downloader.rb, line 133
def log_downloading(url, output_file)
  message = "#{@entity_identifier}: #{output_file} -- Downloading from #{url} ..."
  PushmiPullyu::Logging.log_aip_activity(@aip_directory, message)
end
log_saved(is_success, output_file) click to toggle source
# File lib/pushmi_pullyu/aip/downloader.rb, line 138
def log_saved(is_success, output_file)
  message = "#{@entity_identifier}: #{output_file} -- #{is_success ? 'Saved' : 'Failed'}"
  PushmiPullyu::Logging.log_aip_activity(@aip_directory, message)
end
make_directories() click to toggle source
# File lib/pushmi_pullyu/aip/downloader.rb, line 165
def make_directories
  PushmiPullyu.logger.debug("#{@entity_identifier}: Creating directories ...")
  clean_directories
  aip_dirs.each_value do |path|
    FileUtils.mkdir_p(path)
  end
  PushmiPullyu.logger.debug("#{@entity_identifier}: Creating directories done")
end
make_file_set_directories(file_set_uuid) click to toggle source
# File lib/pushmi_pullyu/aip/downloader.rb, line 174
def make_file_set_directories(file_set_uuid)
  PushmiPullyu.logger.debug("#{@entity_identifier}: Creating file set #{file_set_uuid} directories ...")
  file_set_dirs(file_set_uuid).each_value do |path|
    FileUtils.mkdir_p(path)
  end
  PushmiPullyu.logger.debug("#{@entity_identifier}: Creating file set #{file_set_uuid} directories done")
end
object_aip_paths() click to toggle source

Files

# File lib/pushmi_pullyu/aip/downloader.rb, line 191
def object_aip_paths
  @object_aip_paths ||= {
    # Base path
    main_object_remote: object_uri,
    main_object_local: "#{aip_dirs[:metadata]}/object_metadata.n3",
    file_sets_remote: "#{object_uri}/filesets",
    file_sets_local: "#{aip_dirs[:files_metadata]}/file_order.xml",
    # This is downloaded for processing but not saved
    file_paths_remote: "#{object_uri}/file_paths"
  }.freeze
end
object_uri() click to toggle source
# File lib/pushmi_pullyu/aip/downloader.rb, line 126
def object_uri
  aip_api_url = PushmiPullyu.options[:jupiter][:aip_api_path]
  @object_uri ||= "#{aip_api_url}/#{@entity[:type]}/#{@entity[:uuid]}"
end