class Fs3Cloner::BucketCloner

Attributes

copy_opts[R]
excludes[R]
from_bucket[R]
includes[R]
logger[R]
output[R]
prefixes[R]
to_bucket[R]

Public Class Methods

new(from_credentials, to_credentials, options = {}) click to toggle source

options

  • output : STDOUT

  • includes : ex. [%r(uploads/document), “uploads/1/document.pdf”]

  • excludes : ex. [%r(uploads/1)]

  • prefixes : ex. [“uploads/document”, “uploads/test”] , this will boost to cloning

# File lib/fs3_cloner/bucket_cloner.rb, line 29
def initialize(from_credentials, to_credentials, options = {})
  parse_options(options)

  @from_bucket = bucket_from_credentials(from_credentials)
  @to_bucket   = bucket_from_credentials(to_credentials)
  @logger      = Logger.new(output)
end

Public Instance Methods

clone() click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 37
def clone
  total_sync = 0
  total_skip = 0

  logger.info "Started syncing ..."

  if prefixes.nil?
    sync_count, skip_count = process_objects(from_bucket.objects)

    total_sync += sync_count
    total_skip += skip_count
  else
    prefixes.each do |prefix|
      logger.info "Processing prefix #{prefix} ..."
      sync_count, skip_count = process_objects(from_bucket.objects(prefix: prefix))

      puts sync_count
      puts skip_count
      total_sync += sync_count
      total_skip += skip_count
    end
  end

  logger.info "Done!"
  logger.info "Synced #{total_sync}"
  logger.info "Skipped #{total_skip}"
end

Private Instance Methods

bucket_from_credentials(credentials) click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 124
def bucket_from_credentials(credentials)
  client = Aws::S3::Client.new(
    access_key_id:      credentials[:aws_access_key_id],
    secret_access_key:  credentials[:aws_secret_access_key]
  )

  bucket = Aws::S3::Bucket.new(credentials[:bucket], client: client)

  return bucket if bucket.exists?

  bucket.create
end
match?(regex, string) click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 165
def match?(regex, string)
  !regex.match(string).nil?
end
object_exclude?(object) click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 151
def object_exclude?(object)
  return false if excludes.nil?

  excludes.any? do |pattern|
    if pattern.is_a?(Regexp)
      match?(pattern, object.key)
    elsif pattern.is_a?(String)
      pattern == object.key
    else
      logger.error "Unknown include pattern, #{pattern}"
    end
  end
end
object_include?(object) click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 137
def object_include?(object)
  return true if includes.nil?

  includes.any? do |pattern|
    if pattern.is_a?(Regexp)
      match?(pattern, object.key)
    elsif pattern.is_a?(String)
      pattern == object.key
    else
      logger.error "Unknown include pattern, #{pattern}"
    end
  end
end
object_needs_syncing?(object) click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 116
def object_needs_syncing?(object)
  to_object = to_bucket.object(object.key)

  return false if skip_object?(object)
  return true if !to_object.exists?
  return to_object.etag != object.etag
end
parse_options(options) click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 84
def parse_options(options)
  @output    = options[:output] || STDOUT
  @excludes  = options[:excludes]
  @includes  = options[:includes]
  @prefixes  = options[:prefixes]
  @copy_opts = options[:copy_opts] || {}
end
pp(object) click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 101
def pp(object)
  content_length_in_kb = object.content_length / 1024

  [
    %Q(#{object.key} #{content_length_in_kb}k),
    %Q(#{object.last_modified.strftime("%b %d %Y %H:%M")})
  ].join(' ')
end
process_objects(objects) click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 67
def process_objects(objects)
  sync_count = 0
  skip_count = 0

  objects.each do |object|
    if object_needs_syncing?(object)
      sync(object)
      sync_count += 1
    else
      skip(object)
      skip_count += 1
    end
  end

  [sync_count, skip_count]
end
skip(object) click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 97
def skip(object)
  logger.debug "Skipped #{pp object}"
end
skip_object?(object) click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 110
def skip_object?(object)
  false ||
    !object_include?(object) ||
    object_exclude?(object)
end
sync(object) click to toggle source
# File lib/fs3_cloner/bucket_cloner.rb, line 92
def sync(object)
  logger.debug "Syncing #{pp object}"
  object.copy_to(to_bucket.object(object.key), copy_opts)
end