class Manage::S3

Public Class Methods

new(aws_access_key_id, aws_secret_key_id) click to toggle source

nodoc

# File lib/manage_s3_bucket.rb, line 22
def initialize(aws_access_key_id, aws_secret_key_id)
  @aws_access_key_id = aws_access_key_id
  @aws_secret_key_id = aws_secret_key_id
end

Public Instance Methods

copy_key(_s3, source_path, dest_path, key) click to toggle source
# File lib/manage_s3_bucket.rb, line 74
def copy_key(_s3, source_path, dest_path, key)
  copied = false
  i = 1
  source_bucket, source_directory = extract_path(source_path)
  dest_bucket, dest_directory     = extract_path(dest_path)
  new_key = "#{dest_directory}#{key.gsub(source_directory, '')}"
  while !copied
    begin
      _s3.CopyObject(
        'SourceBucket'      => source_bucket,
        'SourceObject'      => key,
        'DestinationBucket' => dest_bucket,
        'DestinationObject' => new_key
      )
      copied = true
      pp "SUCCESS: Copyed #{key} to #{new_key}"
    rescue
      i += 1
      pp $!
      sleep 10
      pp '#'*200
      pp "ERROR: Trying #{i} to copy #{key} to #{new_key}"
      _s3 = s3
      copy_key(_s3, source_path, dest_path, key)
    end
  end
end
copy_path(source_path, dest_path) click to toggle source
# File lib/manage_s3_bucket.rb, line 102
def copy_path(source_path, dest_path)
  # It's not too fast as s3mirror (java tool)
  max_threads = 500
  wq = WorkQueue.new max_threads
  execute_in_path source_path do |keys|
    wq.enqueue_b do
      _s3 = s3
      keys.each do |key|
        copy_key(_s3, source_path, dest_path, key)
      end
    end
  end
end
execute_in_path(path, marker = nil) { |keys| ... } click to toggle source
# File lib/manage_s3_bucket.rb, line 48
def execute_in_path(path, marker = nil, &block)
  marker ||= nil
  bucket, directory = extract_path(path)
  list = list_directory(path, marker)
  objects = list['Contents']
  if objects
    keys = objects.map { |c| c['Key'] }
    yield keys
    if list["IsTruncated"] == "true"
      marker = keys.last
      p "Next Page: #{marker}"
      execute_in_path path, marker do |keys|
        yield keys
      end
    end
  end
end
extract_path(path) click to toggle source
# File lib/manage_s3_bucket.rb, line 31
def extract_path(path)
  slice_path = path.split('/')
  [slice_path[0], slice_path[1..-1].join('/')]
end
list_directory(path, marker = nil) click to toggle source
# File lib/manage_s3_bucket.rb, line 36
def list_directory(path, marker = nil)
  bucket, directory = extract_path(path)
  opt_list = {
    'Bucket' => bucket,
    'prefix' => directory,
    "max-keys" => 1000
  }
  opt_list['marker'] = marker if marker
  list = s3.ListObjects(opt_list)['ListBucketResult']
  return list
end
remove_path(path) click to toggle source
# File lib/manage_s3_bucket.rb, line 66
def remove_path(path)
  bucket, directory = extract_path(path)
  p "Deleting #{path}"
  execute_in_path path do |keys|
    s3.DeleteMultipleObjects('Bucket' => bucket, 'Object' => keys)
  end
end
s3() click to toggle source
# File lib/manage_s3_bucket.rb, line 27
def s3
  @s3 ||= RightScale::CloudApi::AWS::S3::Manager::new(@aws_access_key_id, @aws_secret_key_id, 'https://s3.amazonaws.com')
end