class DeeBee::CloudSync
Attributes
sync_settings[R]
Public Class Methods
new(configuration = DeeBee::Configuration.new)
click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 14 def initialize (configuration = DeeBee::Configuration.new) @sync_settings = configuration.settings['cloud_sync'] end
Public Instance Methods
execute()
click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 18 def execute puts "\nPerforming cloud sync..." time_elapsed_for("Cloud sync") do local_directory = sync_settings['local_directory'] puts " Syncing directory '#{local_directory}'" puts " to provider '#{sync_settings['credentials']['provider']}'" puts " on storage '#{sync_settings['provider_settings']['remote_storage']}'" Dir.chdir(local_directory) do local_files_metadata = create_local_files_metadata raise 'local file list is empty: aborting' unless file_metadata_contains_files?(local_files_metadata) remote_storage = get_remote_storage remote_directory = get_remote_directory(remote_storage) upload_local_files_to_remote_directory(local_files_metadata, remote_directory) remove_orphaned_remote_files_older_than(local_files_metadata, remote_directory, DEFAULT_DAYS_TO_KEEP_ORPHANS) move_remote_files_to_long_term_storage_directory(remote_storage, local_files_metadata, remote_directory) end end end
Private Instance Methods
copy_local_file_to_remote_directory(filepath, remote_directory)
click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 122 def copy_local_file_to_remote_directory (filepath, remote_directory) remote_directory.files.create( :key => filepath, :public => false, :body => File.open(filepath), :cache_control => "max-age=#{ONE_MONTH}" ) end
create_directory_on_remote_directory(path, remote_directory)
click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 114 def create_directory_on_remote_directory (path, remote_directory) # NOTE - empty objects with keys that end in "/" are created to help clients resolve directories remote_directory.files.create( :key => path, :public => false ) end
create_local_files_metadata()
click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 42 def create_local_files_metadata puts " Creating local file metadata..." local_files_metadata = Dir.glob('**/*').inject({}) do |hsh, path| if File.directory? path hsh.update("#{path}/" => :directory) else hsh.update(path => Digest::MD5.file(path).hexdigest) end end puts " done creating metadata at #{Time.now}" local_files_metadata end
file_metadata_contains_files?(file_metadata)
click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 55 def file_metadata_contains_files? (file_metadata) file_metadata.any?{|path,directory_or_hash| directory_or_hash != :directory} end
get_remote_directory(remote_storage)
click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 63 def get_remote_directory (remote_storage) remote_storage.directories.create(:key => sync_settings['provider_settings']['remote_storage']) remote_storage.directories.get( sync_settings['provider_settings']['remote_storage'] ) end
get_remote_storage()
click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 59 def get_remote_storage Fog::Storage.new( symbolize_keys(sync_settings['credentials']) ) end
identical_file_in_remote_directory?(remote_directory, file, etag)
click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 101 def identical_file_in_remote_directory? (remote_directory, file, etag) remote_directory_file_etag(remote_directory, file) == etag rescue Excon::Errors::NotFound, NoHeadError return false end
move_remote_files_to_long_term_storage_directory(remote_storage, local_files_metadata, remote_directory)
click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 135 def move_remote_files_to_long_term_storage_directory (remote_storage, local_files_metadata, remote_directory) return unless !!sync_settings['long_term_archive'] remote_long_term_storage_directory = sync_settings['long_term_archive']['subdirectory'] puts "\n Moving remote files to long term storage directory" remote_directory.files.each do |remote_object| if !remote_object.content_length.zero? && remote_object.key !~ /^#{remote_long_term_storage_directory}\// object_age_in_days = age_in_days(remote_object.last_modified) if object_age_in_days > sync_settings['long_term_archive']['rotation_age'] age_text = "day#{ object_age_in_days == 1 ? '' : 's' } old" puts " Moving #{remote_object.key} to long term storage (#{object_age_in_days} #{age_text})" remote_storage.copy_object( sync_settings['provider_settings']['remote_storage'], remote_object.key, sync_settings['provider_settings']['remote_storage'], "#{remote_long_term_storage_directory}/#{remote_object.key}" ) remote_object.destroy if local_files_metadata.has_key?(remote_object.key) File.unlink(File.join(sync_settings['local_directory'], remote_object.key)) end end end end end
remote_directory_file_etag(remote_directory, file)
click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 107 def remote_directory_file_etag (remote_directory, file) head = remote_directory.files.head(file) raise NoHeadError if head.nil? head.etag end
remote_files_of_remote_directory(remote_directory)
click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 131 def remote_files_of_remote_directory (remote_directory) remote_directory.files.map{ |file| file } end
remote_object_in_long_term_archive?(remote_object_key)
click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 163 def remote_object_in_long_term_archive? (remote_object_key) return false unless !!sync_settings['long_term_archive'] return false unless !!sync_settings['long_term_archive']['subdirectory'] return remote_object_key =~ /^#{ sync_settings['long_term_archive']['subdirectory'] }\// end
remove_orphaned_remote_files_older_than(local_files_metadata, remote_directory, days_old=DEFAULT_DAYS_TO_KEEP_ORPHANS)
click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 82 def remove_orphaned_remote_files_older_than(local_files_metadata, remote_directory, days_old=DEFAULT_DAYS_TO_KEEP_ORPHANS) puts "\n Checking orphaned remote files" remote_directory.files.each do |remote_object| unless local_files_metadata.has_key? remote_object.key next if remote_object_in_long_term_archive?(remote_object.key) object_age_in_days = age_in_days(remote_object.last_modified) age_text = "day#{ object_age_in_days == 1 ? '' : 's' } old" if object_age_in_days > days_old puts " Removing #{remote_object.key} (no longer exists locally, #{object_age_in_days} #{age_text})" remote_object.destroy else puts " Keeping orphaned remote #{remote_object.key} (#{object_age_in_days} #{age_text})" end end end end
upload_local_files_to_remote_directory(local_files_metadata, remote_directory)
click to toggle source
# File lib/dee_bee/cloud_sync.rb, line 68 def upload_local_files_to_remote_directory (local_files_metadata, remote_directory) local_files_metadata.each do |file, etag| if etag == :directory puts " Directory #{file}" create_directory_on_remote_directory(file, remote_directory) elsif identical_file_in_remote_directory?(remote_directory, file, etag) puts " Skipping #{file} (identical)" else puts " Uploading #{file}" copy_local_file_to_remote_directory(file,remote_directory) end end end