class Fs3Cloner::BucketCloner
Attributes
copy_opts[R]
excludes[R]
from_bucket[R]
includes[R]
logger[R]
output[R]
prefixes[R]
to_bucket[R]
Public Class Methods
new(from_credentials, to_credentials, options = {})
click to toggle source
options
-
output : STDOUT
-
includes : ex. [%r(uploads/document), “uploads/1/document.pdf”]
-
excludes : ex. [%r(uploads/1)]
-
prefixes : ex. [“uploads/document”, “uploads/test”] , this will boost to cloning
# File lib/fs3_cloner/bucket_cloner.rb, line 29 def initialize(from_credentials, to_credentials, options = {}) parse_options(options) @from_bucket = bucket_from_credentials(from_credentials) @to_bucket = bucket_from_credentials(to_credentials) @logger = Logger.new(output) end
Public Instance Methods
clone()
click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 37 def clone total_sync = 0 total_skip = 0 logger.info "Started syncing ..." if prefixes.nil? sync_count, skip_count = process_objects(from_bucket.objects) total_sync += sync_count total_skip += skip_count else prefixes.each do |prefix| logger.info "Processing prefix #{prefix} ..." sync_count, skip_count = process_objects(from_bucket.objects(prefix: prefix)) puts sync_count puts skip_count total_sync += sync_count total_skip += skip_count end end logger.info "Done!" logger.info "Synced #{total_sync}" logger.info "Skipped #{total_skip}" end
Private Instance Methods
bucket_from_credentials(credentials)
click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 124 def bucket_from_credentials(credentials) client = Aws::S3::Client.new( access_key_id: credentials[:aws_access_key_id], secret_access_key: credentials[:aws_secret_access_key] ) bucket = Aws::S3::Bucket.new(credentials[:bucket], client: client) return bucket if bucket.exists? bucket.create end
match?(regex, string)
click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 165 def match?(regex, string) !regex.match(string).nil? end
object_exclude?(object)
click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 151 def object_exclude?(object) return false if excludes.nil? excludes.any? do |pattern| if pattern.is_a?(Regexp) match?(pattern, object.key) elsif pattern.is_a?(String) pattern == object.key else logger.error "Unknown include pattern, #{pattern}" end end end
object_include?(object)
click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 137 def object_include?(object) return true if includes.nil? includes.any? do |pattern| if pattern.is_a?(Regexp) match?(pattern, object.key) elsif pattern.is_a?(String) pattern == object.key else logger.error "Unknown include pattern, #{pattern}" end end end
object_needs_syncing?(object)
click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 116 def object_needs_syncing?(object) to_object = to_bucket.object(object.key) return false if skip_object?(object) return true if !to_object.exists? return to_object.etag != object.etag end
parse_options(options)
click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 84 def parse_options(options) @output = options[:output] || STDOUT @excludes = options[:excludes] @includes = options[:includes] @prefixes = options[:prefixes] @copy_opts = options[:copy_opts] || {} end
pp(object)
click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 101 def pp(object) content_length_in_kb = object.content_length / 1024 [ %Q(#{object.key} #{content_length_in_kb}k), %Q(#{object.last_modified.strftime("%b %d %Y %H:%M")}) ].join(' ') end
process_objects(objects)
click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 67 def process_objects(objects) sync_count = 0 skip_count = 0 objects.each do |object| if object_needs_syncing?(object) sync(object) sync_count += 1 else skip(object) skip_count += 1 end end [sync_count, skip_count] end
skip(object)
click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 97 def skip(object) logger.debug "Skipped #{pp object}" end
skip_object?(object)
click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 110 def skip_object?(object) false || !object_include?(object) || object_exclude?(object) end
sync(object)
click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 92 def sync(object) logger.debug "Syncing #{pp object}" object.copy_to(to_bucket.object(object.key), copy_opts) end