class DeeBee::CloudSync

Attributes

sync_settings[R]

Public Class Methods

new(configuration = DeeBee::Configuration.new) click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 14
def initialize (configuration = DeeBee::Configuration.new)
  @sync_settings =  configuration.settings['cloud_sync']
end

Public Instance Methods

execute() click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 18
def execute
  puts "\nPerforming cloud sync..."
  time_elapsed_for("Cloud sync") do
    local_directory = sync_settings['local_directory']
    puts "  Syncing directory '#{local_directory}'"
    puts "    to provider '#{sync_settings['credentials']['provider']}'"
    puts "    on storage '#{sync_settings['provider_settings']['remote_storage']}'"

    Dir.chdir(local_directory) do
      local_files_metadata = create_local_files_metadata
      raise 'local file list is empty: aborting' unless file_metadata_contains_files?(local_files_metadata)

      remote_storage = get_remote_storage
      remote_directory = get_remote_directory(remote_storage)

      upload_local_files_to_remote_directory(local_files_metadata, remote_directory)
      remove_orphaned_remote_files_older_than(local_files_metadata, remote_directory, DEFAULT_DAYS_TO_KEEP_ORPHANS)
      move_remote_files_to_long_term_storage_directory(remote_storage, local_files_metadata, remote_directory)
    end
  end
end

Private Instance Methods

copy_local_file_to_remote_directory(filepath, remote_directory) click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 122
def copy_local_file_to_remote_directory (filepath, remote_directory)
  remote_directory.files.create(
    :key => filepath,
    :public => false,
    :body => File.open(filepath),
    :cache_control => "max-age=#{ONE_MONTH}"
  )
end
create_directory_on_remote_directory(path, remote_directory) click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 114
def create_directory_on_remote_directory (path, remote_directory)
  # NOTE - empty objects with keys that end in "/" are created to help clients resolve directories
  remote_directory.files.create(
    :key => path,
    :public => false
  )
end
create_local_files_metadata() click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 42
def create_local_files_metadata
  puts "  Creating local file metadata..."
  local_files_metadata = Dir.glob('**/*').inject({}) do |hsh, path|
    if File.directory? path
      hsh.update("#{path}/" => :directory)
    else
      hsh.update(path => Digest::MD5.file(path).hexdigest)
    end
  end
  puts "  done creating metadata at #{Time.now}"
  local_files_metadata
end
file_metadata_contains_files?(file_metadata) click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 55
def file_metadata_contains_files? (file_metadata)
  file_metadata.any?{|path,directory_or_hash| directory_or_hash != :directory}
end
get_remote_directory(remote_storage) click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 63
def get_remote_directory (remote_storage)
  remote_storage.directories.create(:key => sync_settings['provider_settings']['remote_storage'])
  remote_storage.directories.get( sync_settings['provider_settings']['remote_storage'] )
end
get_remote_storage() click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 59
def get_remote_storage
  Fog::Storage.new( symbolize_keys(sync_settings['credentials']) )
end
identical_file_in_remote_directory?(remote_directory, file, etag) click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 101
def identical_file_in_remote_directory? (remote_directory, file, etag)
  remote_directory_file_etag(remote_directory, file) == etag
rescue Excon::Errors::NotFound, NoHeadError
  return false
end
move_remote_files_to_long_term_storage_directory(remote_storage, local_files_metadata, remote_directory) click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 135
def move_remote_files_to_long_term_storage_directory (remote_storage, local_files_metadata, remote_directory)
  return unless !!sync_settings['long_term_archive']

  remote_long_term_storage_directory = sync_settings['long_term_archive']['subdirectory']

  puts "\n  Moving remote files to long term storage directory"
  remote_directory.files.each do |remote_object|
    if !remote_object.content_length.zero? && remote_object.key !~ /^#{remote_long_term_storage_directory}\//
      object_age_in_days = age_in_days(remote_object.last_modified)
      if object_age_in_days > sync_settings['long_term_archive']['rotation_age']
        age_text = "day#{ object_age_in_days == 1 ? '' : 's' } old"
        puts "  Moving #{remote_object.key} to long term storage (#{object_age_in_days} #{age_text})"
        remote_storage.copy_object(
          sync_settings['provider_settings']['remote_storage'],
          remote_object.key,
          sync_settings['provider_settings']['remote_storage'],
          "#{remote_long_term_storage_directory}/#{remote_object.key}"
        )
        remote_object.destroy

        if local_files_metadata.has_key?(remote_object.key)
          File.unlink(File.join(sync_settings['local_directory'], remote_object.key))
        end
      end
    end
  end
end
remote_directory_file_etag(remote_directory, file) click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 107
def remote_directory_file_etag (remote_directory, file)
  head = remote_directory.files.head(file)
  raise NoHeadError if head.nil?

  head.etag
end
remote_files_of_remote_directory(remote_directory) click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 131
def remote_files_of_remote_directory (remote_directory)
  remote_directory.files.map{ |file| file }
end
remote_object_in_long_term_archive?(remote_object_key) click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 163
def remote_object_in_long_term_archive? (remote_object_key)
  return false unless !!sync_settings['long_term_archive']
  return false unless !!sync_settings['long_term_archive']['subdirectory']

  return remote_object_key =~ /^#{ sync_settings['long_term_archive']['subdirectory'] }\//
end
remove_orphaned_remote_files_older_than(local_files_metadata, remote_directory, days_old=DEFAULT_DAYS_TO_KEEP_ORPHANS) click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 82
def remove_orphaned_remote_files_older_than(local_files_metadata, remote_directory, days_old=DEFAULT_DAYS_TO_KEEP_ORPHANS)
  puts "\n  Checking orphaned remote files"
  remote_directory.files.each do |remote_object|
    unless local_files_metadata.has_key? remote_object.key
      next if remote_object_in_long_term_archive?(remote_object.key)

      object_age_in_days = age_in_days(remote_object.last_modified)
      age_text = "day#{ object_age_in_days == 1 ? '' : 's' } old"

      if object_age_in_days > days_old
        puts "  Removing #{remote_object.key} (no longer exists locally, #{object_age_in_days} #{age_text})"
        remote_object.destroy
      else
        puts "  Keeping orphaned remote #{remote_object.key} (#{object_age_in_days} #{age_text})"
      end
    end
  end
end
upload_local_files_to_remote_directory(local_files_metadata, remote_directory) click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 68
def upload_local_files_to_remote_directory (local_files_metadata, remote_directory)
  local_files_metadata.each do |file, etag|
    if etag == :directory
      puts "  Directory #{file}"
      create_directory_on_remote_directory(file, remote_directory)
    elsif identical_file_in_remote_directory?(remote_directory, file, etag)
      puts "  Skipping #{file} (identical)"
    else
      puts "  Uploading #{file}"
      copy_local_file_to_remote_directory(file,remote_directory)
    end
  end
end