class Manage::S3
Public Class Methods
new(aws_access_key_id, aws_secret_key_id)
click to toggle source
nodoc
# File lib/manage_s3_bucket.rb, line 22 def initialize(aws_access_key_id, aws_secret_key_id) @aws_access_key_id = aws_access_key_id @aws_secret_key_id = aws_secret_key_id end
Public Instance Methods
copy_key(_s3, source_path, dest_path, key)
click to toggle source
# File lib/manage_s3_bucket.rb, line 74 def copy_key(_s3, source_path, dest_path, key) copied = false i = 1 source_bucket, source_directory = extract_path(source_path) dest_bucket, dest_directory = extract_path(dest_path) new_key = "#{dest_directory}#{key.gsub(source_directory, '')}" while !copied begin _s3.CopyObject( 'SourceBucket' => source_bucket, 'SourceObject' => key, 'DestinationBucket' => dest_bucket, 'DestinationObject' => new_key ) copied = true pp "SUCCESS: Copyed #{key} to #{new_key}" rescue i += 1 pp $! sleep 10 pp '#'*200 pp "ERROR: Trying #{i} to copy #{key} to #{new_key}" _s3 = s3 copy_key(_s3, source_path, dest_path, key) end end end
copy_path(source_path, dest_path)
click to toggle source
# File lib/manage_s3_bucket.rb, line 102 def copy_path(source_path, dest_path) # It's not too fast as s3mirror (java tool) max_threads = 500 wq = WorkQueue.new max_threads execute_in_path source_path do |keys| wq.enqueue_b do _s3 = s3 keys.each do |key| copy_key(_s3, source_path, dest_path, key) end end end end
execute_in_path(path, marker = nil) { |keys| ... }
click to toggle source
# File lib/manage_s3_bucket.rb, line 48 def execute_in_path(path, marker = nil, &block) marker ||= nil bucket, directory = extract_path(path) list = list_directory(path, marker) objects = list['Contents'] if objects keys = objects.map { |c| c['Key'] } yield keys if list["IsTruncated"] == "true" marker = keys.last p "Next Page: #{marker}" execute_in_path path, marker do |keys| yield keys end end end end
extract_path(path)
click to toggle source
# File lib/manage_s3_bucket.rb, line 31 def extract_path(path) slice_path = path.split('/') [slice_path[0], slice_path[1..-1].join('/')] end
list_directory(path, marker = nil)
click to toggle source
# File lib/manage_s3_bucket.rb, line 36 def list_directory(path, marker = nil) bucket, directory = extract_path(path) opt_list = { 'Bucket' => bucket, 'prefix' => directory, "max-keys" => 1000 } opt_list['marker'] = marker if marker list = s3.ListObjects(opt_list)['ListBucketResult'] return list end
remove_path(path)
click to toggle source
# File lib/manage_s3_bucket.rb, line 66 def remove_path(path) bucket, directory = extract_path(path) p "Deleting #{path}" execute_in_path path do |keys| s3.DeleteMultipleObjects('Bucket' => bucket, 'Object' => keys) end end
s3()
click to toggle source
# File lib/manage_s3_bucket.rb, line 27 def s3 @s3 ||= RightScale::CloudApi::AWS::S3::Manager::new(@aws_access_key_id, @aws_secret_key_id, 'https://s3.amazonaws.com') end