class Mamiya::Storages::S3Proxy

Because there's no S3 endpoint in Amazon VPC, fetching from instances with no public IP may consume your NAT instances' bandwidth. This basically uses Amazon S3 but use s3_proxy.gem for fetching from specific host to avoid heavy bandwidth load.

Note: s3_proxy is a simple Rack app that proxies HTTP GET requests to Amazon S3 GetObject. You can use it with puma + nginx proxy_cache to avoid heavy bandwidth load.

Public Instance Methods

fetch(package_name, dir) click to toggle source
# File lib/mamiya/storages/s3_proxy.rb, line 16
def fetch(package_name, dir)
  package_key, meta_key = package_and_meta_key_for(package_name)

  package_path = File.join(dir, File.basename(package_key))
  meta_path = File.join(dir, File.basename(meta_key))

  if File.exists?(package_path) && File.exists?(meta_path)
    raise AlreadyFetched
  end

  tmp_package_path = "#{package_path}.progress"
  tmp_meta_path = "#{meta_path}.progress"
  open(tmp_package_path, 'wb+') do |io|
    proxy_get(package_key, io)
  end
  open(tmp_meta_path, 'wb+') do |io|
    proxy_get(meta_key, io)
  end
  FileUtils.mv(tmp_package_path, package_path)
  FileUtils.mv(tmp_meta_path, meta_path)

  return Mamiya::Package.new(package_path)

rescue NotFound, AlreadyFetched => e
  raise e

rescue Exception => e
  File.unlink package_path if package_path && File.exists?(package_path)
  File.unlink meta_path if meta_path && File.exists?(meta_path)

  raise e
end
meta(package_name) click to toggle source
# File lib/mamiya/storages/s3_proxy.rb, line 49
def meta(package_name)
  _, meta_key = package_and_meta_key_for(package_name)
  JSON.parse(proxy_get(meta_key, nil, &:body))
rescue NotFound
  return nil
end

Private Instance Methods

connect_proxy(&block) click to toggle source
# File lib/mamiya/storages/s3_proxy.rb, line 69
def connect_proxy(&block)
  Net::HTTP.new(proxy_host_uri.host, proxy_host_uri.port).tap do |http|
    http.use_ssl = (proxy_host_uri.scheme == 'https')
    if @config[:proxy_ssl_verify_none] == 'none'
      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
    end

    return http.start(&block)
  end
end
proxy_get(key, target) { |response| ... } click to toggle source
# File lib/mamiya/storages/s3_proxy.rb, line 80
def proxy_get(key, target)
  connect_proxy do |http|
    http.request_get("#{proxy_host_uri.path}/#{@config[:bucket]}/#{key}") do |response|
      response.value

      if block_given?
        return yield(response)
      else
        response.read_body do |chunk|
          target.write chunk
        end
      end
    end
  end
rescue Net::HTTPServerException => e
  if e.message.start_with?('404 ')
    raise NotFound
  else
    raise e
  end
end
proxy_host_uri() click to toggle source
# File lib/mamiya/storages/s3_proxy.rb, line 65
def proxy_host_uri
  @proxy_host_uri ||= URI.parse(@config[:proxy_host])
end