class Rack2Aws::FileManager

Attributes

allow_public[R]
aws[R]
aws_directory[R]
files[R]
nproc[R]
per_page[R]
rackspace[R]
rackspace_directory[R]
total[R]
verbose_mode[R]

Public Class Methods

new(options={}) click to toggle source
# File lib/rack2aws.rb, line 16
def initialize(options={})
  options = default_options.merge(options)
  @per_page = options[:per_page]
  @rackspace = Fog::Storage.new(RackspaceConfig.load())
  @aws = Fog::Storage.new(AWSConfig.load())
  @rackspace_directory = rackspace.directories.get(options[:container])
  @aws_directory = aws.directories.get(options[:bucket])
  @nproc = options[:nproc]
  @allow_public = options[:public]
  @verbose_mode = options[:verbose]
  @files = []
  @total = 0
end

Public Instance Methods

copy() click to toggle source
# File lib/rack2aws.rb, line 34
def copy
  time = Time.now
  pages = rackspace_directory.count / per_page + 1
  marker = ''

  # get Rackspace files
  pages.times do |i|
    puts "! Getting page #{i+1}..."
    files = rackspace_directory.files.all(:limit => per_page, :marker => marker).to_a
    puts "! #{files.size} files in page #{i+1}, forking..." if verbose_mode
    pid = fork do
      copy_files(i, files)
    end
    puts "! Process #{pid} forked to copy files" if verbose_mode
    marker = files.last.key
    @total += files.size
  end

  pages.times do
    Process.wait
  end

  puts "--------------------------------------------------"
  puts "! #{total} files copied in #{Time.now - time}secs."
  puts "--------------------------------------------------\n\n"
end
default_options() click to toggle source
# File lib/rack2aws.rb, line 30
def default_options
  { :per_page => 10000, :public => false, :verbose => false }
end

Private Instance Methods

copy_file(file) click to toggle source
# File lib/rack2aws.rb, line 87
def copy_file(file)
  aws_directory.files.create(:key          => file.key,
                             :body         => file.body,
                             :content_type => file.content_type,
                             :public       => allow_public)
end
copy_files(page, files) click to toggle source
# File lib/rack2aws.rb, line 61
def copy_files(page, files)
  puts "  [#{Process.pid}] Page #{page+1}: Copying #{files.size} files..." if verbose_mode
  total = files.size
  process_pids = {}
  time = Time.now

  while !files.empty? or !process_pids.empty?
    while process_pids.size < nproc and files.any? do
      file = files.pop
      pid = Process.fork do
        copy_file(file)
        exit!(0)
      end
      process_pids[pid] = { :file => file }
    end

    if pid_done = Process.wait
      if job_finished = process_pids.delete(pid_done)
        puts "    [#{Process.pid}] Page #{page+1}: Copied #{job_finished[:file].key}." if verbose_mode
      end
    end
  end

  puts "  [#{Process.pid}] ** Page #{page+1}: Copied #{total} files in #{Time.now - time}secs" if verbose_mode
end