class Aws::S3::FileDownloader

@api private

Constants

MAX_PARTS
MIN_CHUNK_SIZE
THREAD_COUNT

Attributes

client[R]

@return [Client]

Public Class Methods

new(options = {}) click to toggle source
# File lib/aws-sdk-s3/file_downloader.rb, line 17
def initialize(options = {})
  @client = options[:client] || Client.new
end

Public Instance Methods

download(destination, options = {}) click to toggle source
# File lib/aws-sdk-s3/file_downloader.rb, line 24
def download(destination, options = {})
  @path = destination
  @mode = options[:mode] || 'auto'
  @thread_count = options[:thread_count] || THREAD_COUNT
  @chunk_size = options[:chunk_size]
  @params = {
    bucket: options[:bucket],
    key: options[:key],
  }
  @params[:version_id] = options[:version_id] if options[:version_id]

  case @mode
  when 'auto' then multipart_download
  when 'single_request' then single_request
  when 'get_range'
    if @chunk_size
      resp = @client.head_object(@params)
      multithreaded_get_by_ranges(construct_chunks(resp.content_length))
    else
      msg = 'In :get_range mode, :chunk_size must be provided'
      raise ArgumentError, msg
    end
  else
    msg = "Invalid mode #{@mode} provided, "\
          'mode should be :single_request, :get_range or :auto'
    raise ArgumentError, msg
  end
end

Private Instance Methods

batches(chunks, mode) click to toggle source
# File lib/aws-sdk-s3/file_downloader.rb, line 106
def batches(chunks, mode)
  chunks = (1..chunks) if mode.eql? 'part_number'
  chunks.each_slice(@thread_count).to_a
end
compute_chunk(file_size) click to toggle source
# File lib/aws-sdk-s3/file_downloader.rb, line 93
def compute_chunk(file_size)
  if @chunk_size && @chunk_size > file_size
    raise ArgumentError, ":chunk_size shouldn't exceed total file size."
  else
    chunk_size = @chunk_size || [
      (file_size.to_f / MAX_PARTS).ceil,
      MIN_CHUNK_SIZE
    ].max.to_i
    chunk_size -= 1 if file_size % chunk_size == 1
    chunk_size
  end
end
compute_mode(file_size, count) click to toggle source
# File lib/aws-sdk-s3/file_downloader.rb, line 71
def compute_mode(file_size, count)
  chunk_size = compute_chunk(file_size)
  part_size = (file_size.to_f / count.to_f).ceil
  if chunk_size < part_size
    multithreaded_get_by_ranges(construct_chunks(file_size))
  else
    multithreaded_get_by_parts(count)
  end
end
construct_chunks(file_size) click to toggle source
# File lib/aws-sdk-s3/file_downloader.rb, line 81
def construct_chunks(file_size)
  offset = 0
  default_chunk_size = compute_chunk(file_size)
  chunks = []
  while offset <= file_size
    progress = offset + default_chunk_size
    chunks << "bytes=#{offset}-#{progress < file_size ? progress : file_size}"
    offset = progress + 1
  end
  chunks
end
multipart_download() click to toggle source
# File lib/aws-sdk-s3/file_downloader.rb, line 55
def multipart_download
  resp = @client.head_object(@params.merge(part_number: 1))
  count = resp.parts_count
  if count.nil? || count <= 1
    resp.content_length < MIN_CHUNK_SIZE ?
      single_request :
      multithreaded_get_by_ranges(construct_chunks(resp.content_length))
  else
    # partNumber is an option
    resp = @client.head_object(@params)
    resp.content_length < MIN_CHUNK_SIZE ?
      single_request :
      compute_mode(resp.content_length, count)
  end
end
multithreaded_get_by_parts(parts) click to toggle source
# File lib/aws-sdk-s3/file_downloader.rb, line 115
def multithreaded_get_by_parts(parts)
  thread_batches(parts, 'part_number')
end
multithreaded_get_by_ranges(chunks) click to toggle source
# File lib/aws-sdk-s3/file_downloader.rb, line 111
def multithreaded_get_by_ranges(chunks)
  thread_batches(chunks, 'range')
end
single_request() click to toggle source
# File lib/aws-sdk-s3/file_downloader.rb, line 140
def single_request
  @client.get_object(
    @params.merge(response_target: @path)
  )
end
thread_batches(chunks, param) click to toggle source
# File lib/aws-sdk-s3/file_downloader.rb, line 119
def thread_batches(chunks, param)
  batches(chunks, param).each do |batch|
    threads = []
    batch.each do |chunk|
      threads << Thread.new do
        resp = @client.get_object(
          @params.merge(param.to_sym => chunk)
        )
        write(resp)
      end
    end
    threads.each(&:join)
  end
end
write(resp) click to toggle source
# File lib/aws-sdk-s3/file_downloader.rb, line 134
def write(resp)
  range, _ = resp.content_range.split(' ').last.split('/')
  head, _ = range.split('-').map {|s| s.to_i}
  IO.write(@path, resp.body.read, head)
end