class RestFtpDaemon::Remote::RemoteS3
Constants
- S3_MAX_COUNT
- S3_MIN_PART
Defaults
Attributes
client[R]
Class options
target[R]
Public Class Methods
handles()
click to toggle source
URI
schemes handled by this plugin
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 23 def self.handles [URI::S3] end
new(target, job, config)
click to toggle source
Calls superclass method
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 27 def initialize target, job, config super end
Public Instance Methods
connect()
click to toggle source
Calls superclass method
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 31 def connect super # Connect remote server @client = Aws::S3::Client.new( region: @target.aws_region, credentials: Aws::Credentials.new(@target.aws_id, @target.aws_secret), http_wire_trace: debug_enabled ) rescue Exception => exception raise RemoteConnectError, "#{exception.class}: #{exception.message}" end
connected?()
click to toggle source
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 84 def connected? !@client.nil? end
move(source, target)
click to toggle source
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 68 def move source, target # Identify the source object # obj = @client.bucket(source.aws_bucket).object(source.path_rel) # raise RestFtpDaemon::AssertionFailed, "move: object not found" unless obj # Move the file # log_debug "move: copy bucket[#{source.aws_bucket}] source[#{source.path_rel}] target[#{target.path_rel}]" @client.copy_object(bucket: source.aws_bucket, key: target.path_rel, copy_source: "#{source.aws_bucket}/#{source.path_rel}") # log_debug "move: delete bucket[#{source.aws_bucket}] source[#{source.path_rel}]" @client.delete_object(bucket: source.aws_bucket, key: source.path_rel) # log_debug "move: done" # Move the file # obj.move_to(target.path_rel, :bucket_name => target.aws_bucket) end
push(source, target, &callback)
click to toggle source
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 54 def push source, target, &callback # Push init raise RestFtpDaemon::AssertionFailed, "push/client" if @client.nil? # Do the transfer, passing the file to the best method File.open(source.path_abs, 'r', encoding: 'BINARY') do |file| if file.size >= S3_MIN_PART upload_multipart file, target.aws_bucket, target.path_rel, target.name, &callback else upload_onefile file, target.aws_bucket, target.path_rel, target.name, &callback end end end
size_if_exists(target)
click to toggle source
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 45 def size_if_exists target log_debug "size_if_exists rel[#{target.path_rel}]" object = @client.get_object(bucket: target.aws_bucket, key: target.path_rel) rescue Aws::S3::Errors::NotFound => e return false else return object.content_length end
Private Instance Methods
compute_parts_size(filesize)
click to toggle source
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 159 def compute_parts_size filesize # Initial part size is minimal partsize_mini = S3_MIN_PART # Other partsize if too many blocks partsize_bigf = (filesize.to_f / S3_MAX_COUNT).ceil # Decide return [partsize_mini, partsize_bigf].max end
debug_enabled()
click to toggle source
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 170 def debug_enabled @config[:debug_s3] end
upload_multipart(file, s3_bucket, s3_path, s3_name) { |part_size, s3_name| ... }
click to toggle source
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 98 def upload_multipart file, s3_bucket, s3_path, s3_name, &callback # Init current_part = 1 # Compute parameters file_size = file.size parts_size = compute_parts_size(file_size) parts_count = (file_size.to_f / parts_size).ceil # Prepare basic opts options = { bucket: s3_bucket, key: s3_path, } # Declare multipart upload mpu_create_response = @client.create_multipart_upload(options) options[:upload_id] = mpu_create_response.upload_id log_debug "push: create_multipart_upload", { s3_bucket: s3_bucket, s3_path: s3_path, upload_id: options[:upload_id], file_size: format_bytes(file_size, "B"), parts_size: format_bytes(parts_size, "B"), parts_count: parts_count } # Upload each part file.each_part(parts_size) do |part| # Prepare part upload opts = options.merge({ body: part, part_number: current_part, }) part_size = part.bytesize log_debug "upload_part [#{current_part}/#{parts_count}] part_size[#{part_size}]" # Push this over there resp = @client.upload_part(opts) # Send progress info upwards yield part_size, s3_name # Increment part number current_part += 1 end # Retrieve parts and complete upload parts_resp = @client.list_parts(options) those_parts = parts_resp.parts.map do |part| { part_number: part.part_number, etag: part.etag } end opts = options.merge({ multipart_upload: { parts: those_parts } }) log_debug "complete_multipart_upload" mpu_complete_response = @client.complete_multipart_upload(opts) end
upload_onefile(file, s3_bucket, s3_path, s3_name, &callback)
click to toggle source
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 90 def upload_onefile file, s3_bucket, s3_path, s3_name, &callback log_debug "push: put_object", { s3_bucket: s3_bucket, s3_path: s3_path, } @client.put_object(bucket: s3_bucket, key: s3_path, body: file) end