class Tumugi::Plugin::GoogleCloudStorage::FileSystem
Attributes
client[R]
Public Class Methods
new(config)
click to toggle source
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 14 def initialize(config) save_config(config) end
Public Instance Methods
bucket_exist?(bucket)
click to toggle source
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 211 def bucket_exist?(bucket) client.get_bucket(bucket, options: request_options) true rescue => e return false if e.status_code == 404 process_error(e) end
copy(src_path, dest_path, raise_if_exist: false)
click to toggle source
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 153 def copy(src_path, dest_path, raise_if_exist: false) if raise_if_exist && exist?(dest_path) raise Tumugi::FileAlreadyExistError.new("Path #{dest_path} is already exist") end src_bucket, src_key = path_to_bucket_and_key(src_path) dest_bucket, dest_key = path_to_bucket_and_key(dest_path) if directory?(src_path) src_prefix = add_path_delimiter(src_key) dest_prefix = add_path_delimiter(dest_key) src_path = add_path_delimiter(src_path) copied_objs = [] entries(src_path).each do |entry| suffix = entry.name[src_prefix.length..-1] client.copy_object(src_bucket, src_prefix + suffix, dest_bucket, dest_prefix + suffix, options: request_options) copied_objs << (dest_prefix + suffix) end wait_until { copied_objs.all? {|obj| obj_exist?(dest_bucket, obj)} } else client.copy_object(src_bucket, src_key, dest_bucket, dest_key, options: request_options) wait_until { obj_exist?(dest_bucket, dest_key) } end rescue process_error($!) end
create_bucket(bucket)
click to toggle source
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 188 def create_bucket(bucket) unless bucket_exist?(bucket) b = Google::Apis::StorageV1::Bucket.new(name: bucket) client.insert_bucket(@project_id, b, options: request_options) true else false end rescue process_error($!) end
directory?(path)
click to toggle source
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 75 def directory?(path) bucket, key = path_to_bucket_and_key(path) if root?(key) bucket_exist?(bucket) else obj = add_path_delimiter(key) if obj_exist?(bucket, obj) true else # Any objects with this prefix objects = client.list_objects(bucket, prefix: obj, max_results: 20, options: request_options) !!(objects.items && objects.items.size > 0) end end rescue process_error($!) end
download(path, download_path: nil, mode: 'r', &block)
click to toggle source
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 131 def download(path, download_path: nil, mode: 'r', &block) bucket, key = path_to_bucket_and_key(path) if download_path.nil? download_path = Tempfile.new('tumugi_gcs_file_system').path end client.get_object(bucket, key, download_dest: download_path, options: request_options) wait_until { File.exist?(download_path) } if block_given? File.open(download_path, mode, &block) else File.open(download_path, mode) end rescue process_error($!) end
entries(path)
click to toggle source
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 93 def entries(path) bucket, key = path_to_bucket_and_key(path) obj = add_path_delimiter(key) results = [] next_page_token = '' until next_page_token.nil? objects = client.list_objects(bucket, prefix: obj, page_token: next_page_token, options: request_options) if objects && objects.items results.concat(objects.items) next_page_token = objects.next_page_token else next_page_token = nil end end results rescue process_error($!) end
exist?(path)
click to toggle source
FileSystem
interfaces
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 22 def exist?(path) bucket, key = path_to_bucket_and_key(path) if obj_exist?(bucket, key) true else directory?(path) end rescue process_error($!) end
mkdir(path, parents: true, raise_if_exist: false)
click to toggle source
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 59 def mkdir(path, parents: true, raise_if_exist: false) if exist?(path) if raise_if_exist raise Tumugi::FileAlreadyExistError.new("Path #{path} is already exist") elsif !directory?(path) raise Tumugi::NotADirectoryError.new("Path #{path} is not a directory") end false else put_string("", add_path_delimiter(path)) true end rescue process_error($!) end
move(src_path, dest_path, raise_if_exist: false)
click to toggle source
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 113 def move(src_path, dest_path, raise_if_exist: false) copy(src_path, dest_path, raise_if_exist: raise_if_exist) remove(src_path) end
path_to_bucket_and_key(path)
click to toggle source
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 182 def path_to_bucket_and_key(path) uri = URI.parse(path) raise Tumugi::FileSystemError.new("URI scheme must be 'gs' but '#{uri.scheme}'") unless uri.scheme == 'gs' [ uri.host, uri.path[1..-1] ] end
put_string(contents, path, content_type: 'text/plain')
click to toggle source
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 148 def put_string(contents, path, content_type: 'text/plain') media = StringIO.new(contents) upload(media, path, content_type: content_type) end
remove(path, recursive: true)
click to toggle source
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 33 def remove(path, recursive: true) bucket, key = path_to_bucket_and_key(path) raise Tumugi::FileSystemError.new("Cannot delete root of bucket at path '#{path}'") if root?(key) if obj_exist?(bucket, key) client.delete_object(bucket, key, options: request_options) wait_until { !obj_exist?(bucket, key) } true elsif directory?(path) raise Tumugi::FileSystemError.new("Path '#{path}' is a directory. Must use recursive delete") if !recursive objs = entries(path).map(&:name) client.batch do |client| objs.each do |obj| client.delete_object(bucket, obj, options: request_options) end end wait_until { !directory?(path) } true else false end rescue process_error($!) end
remove_bucket(bucket)
click to toggle source
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 200 def remove_bucket(bucket) if bucket_exist?(bucket) client.delete_bucket(bucket, options: request_options) true else false end rescue process_error($!) end
upload(media, path, content_type: nil)
click to toggle source
Specific methods
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 122 def upload(media, path, content_type: nil) bucket, key = path_to_bucket_and_key(path) obj = Google::Apis::StorageV1::Object.new(bucket: bucket, name: key) client.insert_object(bucket, obj, upload_source: media, content_type: content_type, options: request_options) wait_until { obj_exist?(bucket, key) } rescue process_error($!) end
Private Instance Methods
add_path_delimiter(key)
click to toggle source
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 233 def add_path_delimiter(key) if key.end_with?('/') key else "#{key}/" end end
obj_exist?(bucket, key)
click to toggle source
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 221 def obj_exist?(bucket, key) client.get_object(bucket, key, options: request_options) true rescue => e return false if e.status_code == 404 process_error(e) end
process_error(err)
click to toggle source
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 293 def process_error(err) if err.respond_to?(:body) begin jobj = JSON.parse(err.body) error = jobj["error"] reason = error["errors"].map{|e| e["reason"]}.join(",") errors = error["errors"].map{|e| e["message"] }.join("\n") rescue JSON::ParserError reason = err.status_code.to_s errors = "HTTP Status: #{err.status_code}\nHeaders: #{err.header.inspect}\nBody:\n#{err.body}" end raise Tumugi::FileSystemError.new(errors, reason) else raise err end end
request_options()
click to toggle source
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 280 def request_options { retries: 5, timeout_sec: 60 } end
root?(key)
click to toggle source
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 229 def root?(key) key.nil? || key == '' end
save_config(config)
click to toggle source
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 241 def save_config(config) if config.private_key_file.nil? @project_id = config.project_id client_email = config.client_email private_key = config.private_key else json = JSON.parse(File.read(config.private_key_file)) @project_id = json['project_id'] client_email = json['client_email'] private_key = json['private_key'] end @key = { client_email: client_email, private_key: private_key } end
wait_until(&block)
click to toggle source
# File lib/tumugi/plugin/google_cloud_storage/file_system.rb, line 287 def wait_until(&block) while not block.call sleep 1 end end