module Hdfs
Constants
- HADOOP_HOME
- JAR_PATTERN_0_20
- VERSION
Public Class Methods
@private
# File lib/hdfs_jruby.rb, line 249 def _conv(stat) file_info = {} file_info['path'] = stat.getPath.to_s file_info['length'] = stat.getLen.to_i file_info['modificationTime'] = stat.getModificationTime.to_i file_info['owner'] = stat.getOwner.to_s file_info['group'] = stat.getGroup.to_s file_info['permission'] = stat.getPermission.toShort.to_i file_info['type'] = !stat.isDir ? 'FILE': 'DIRECTORY' return file_info end
@private
# File lib/hdfs_jruby.rb, line 241 def _path(path) if path.nil? raise "path is nil" end Path.new(path) end
@private
# File lib/hdfs_jruby.rb, line 54 def connectAsUser(user) uri = Hdfs::FileSystem.getDefaultUri(@conf) @fs.close if ! @fs.nil? @fs = Hdfs::FileSystem.get(uri, @conf, user) end
delete
@param [String] path @param [Boolean] r recursive false or true (default: false)
# File lib/hdfs_jruby.rb, line 146 def delete(path, r=false) @fs.delete(_path(path), r) end
@return [Boolean] true: directory, false: file
# File lib/hdfs_jruby.rb, line 156 def directory?(path) @fs.isDirectory(_path(path)) end
@param [String] path
# File lib/hdfs_jruby.rb, line 132 def exists?(path) @fs.exists(_path(path)) end
@return [Boolean] true: file, false: directory
# File lib/hdfs_jruby.rb, line 151 def file?(path) @fs.isFile(_path(path)) end
get file or directory from hdfs @param [String] remote surouce (hdfs path) @param [String] local destination (local path)
# File lib/hdfs_jruby.rb, line 181 def get(remote, local) @fs.copyToLocalFile(Path.new(remote), Path.new(local)) end
# File lib/hdfs_jruby.rb, line 215 def get_fs @fs end
get home directory
# File lib/hdfs_jruby.rb, line 186 def get_home_directory() @fs.getHomeDirectory() end
get working directory
# File lib/hdfs_jruby.rb, line 191 def get_working_directory() @fs.getWorkingDirectory() end
@private
# File lib/hdfs_jruby.rb, line 106 def list(path, opts={}) use_glob = opts[:glob] ? true : false p = _path(path) list = nil if use_glob list = @fs.globStatus(p) else list = @fs.listStatus(p) end return [] if list.nil? if ! block_given? ret_list = [] list.each do | stat | ret_list << _conv(stat) end return ret_list else list.each do | stat | yield _conv(stat) end end end
ls @example
Hdfs.ls("hoge/").each do | stat | p stat end
@param [String] path @return [Array] file status array
@note file status:
path length modificationTime owner group permission type
# File lib/hdfs_jruby.rb, line 76 def ls(path) p = _path(path) list = @fs.globStatus(p) return [] if list.nil? ret_list = [] list.each do |stat| if stat.isDir sub_list = @fs.listStatus(stat.getPath) next if sub_list.nil? sub_list.each do | s | if block_given? yield _conv(s) else ret_list << _conv(s) end end else if block_given? yield _conv(stat) else ret_list << _conv(stat) end end end ret_list if ! block_given? end
make directory @param [String] path
# File lib/hdfs_jruby.rb, line 167 def mkdir(path) @fs.mkdirs(_path(path)) end
@param [String] src hdfs source path @param [String] dst hdfs destination path
# File lib/hdfs_jruby.rb, line 138 def move(src, dst) @fs.rename(Path.new(src), Path.new(dst)) end
put file or directory to hdfs @param [String] local surouce (local path) @param [String] remote destination (hdfs path)
# File lib/hdfs_jruby.rb, line 174 def put(local, remote) @fs.copyFromLocalFile(Path.new(local), Path.new(remote)) end
set owner & group @param [String] path @param [String] owner @param [String] group
# File lib/hdfs_jruby.rb, line 211 def set_owner(path, owner, group) @fs.setOwner(_path(path), owner, group) end
set permission @param [String] path @param [Integer] perm permission
# File lib/hdfs_jruby.rb, line 203 def set_permission(path, perm) @fs.setPermission(_path(path), org.apache.hadoop.fs.permission.FsPermission.new(perm)) end
set working directory
# File lib/hdfs_jruby.rb, line 196 def set_working_directory(path) @fs.setWorkingDirectory(_path()) end
@return [Integer] file size
# File lib/hdfs_jruby.rb, line 161 def size(path) @fs.getFileStatus(_path(path)).getLen() end
Private Instance Methods
@private
# File lib/hdfs_jruby.rb, line 249 def _conv(stat) file_info = {} file_info['path'] = stat.getPath.to_s file_info['length'] = stat.getLen.to_i file_info['modificationTime'] = stat.getModificationTime.to_i file_info['owner'] = stat.getOwner.to_s file_info['group'] = stat.getGroup.to_s file_info['permission'] = stat.getPermission.toShort.to_i file_info['type'] = !stat.isDir ? 'FILE': 'DIRECTORY' return file_info end
@private
# File lib/hdfs_jruby.rb, line 241 def _path(path) if path.nil? raise "path is nil" end Path.new(path) end
@private
# File lib/hdfs_jruby.rb, line 54 def connectAsUser(user) uri = Hdfs::FileSystem.getDefaultUri(@conf) @fs.close if ! @fs.nil? @fs = Hdfs::FileSystem.get(uri, @conf, user) end
delete
@param [String] path @param [Boolean] r recursive false or true (default: false)
# File lib/hdfs_jruby.rb, line 146 def delete(path, r=false) @fs.delete(_path(path), r) end
@return [Boolean] true: directory, false: file
# File lib/hdfs_jruby.rb, line 156 def directory?(path) @fs.isDirectory(_path(path)) end
@param [String] path
# File lib/hdfs_jruby.rb, line 132 def exists?(path) @fs.exists(_path(path)) end
@return [Boolean] true: file, false: directory
# File lib/hdfs_jruby.rb, line 151 def file?(path) @fs.isFile(_path(path)) end
get file or directory from hdfs @param [String] remote surouce (hdfs path) @param [String] local destination (local path)
# File lib/hdfs_jruby.rb, line 181 def get(remote, local) @fs.copyToLocalFile(Path.new(remote), Path.new(local)) end
# File lib/hdfs_jruby.rb, line 215 def get_fs @fs end
get home directory
# File lib/hdfs_jruby.rb, line 186 def get_home_directory() @fs.getHomeDirectory() end
get working directory
# File lib/hdfs_jruby.rb, line 191 def get_working_directory() @fs.getWorkingDirectory() end
@private
# File lib/hdfs_jruby.rb, line 106 def list(path, opts={}) use_glob = opts[:glob] ? true : false p = _path(path) list = nil if use_glob list = @fs.globStatus(p) else list = @fs.listStatus(p) end return [] if list.nil? if ! block_given? ret_list = [] list.each do | stat | ret_list << _conv(stat) end return ret_list else list.each do | stat | yield _conv(stat) end end end
ls @example
Hdfs.ls("hoge/").each do | stat | p stat end
@param [String] path @return [Array] file status array
@note file status:
path length modificationTime owner group permission type
# File lib/hdfs_jruby.rb, line 76 def ls(path) p = _path(path) list = @fs.globStatus(p) return [] if list.nil? ret_list = [] list.each do |stat| if stat.isDir sub_list = @fs.listStatus(stat.getPath) next if sub_list.nil? sub_list.each do | s | if block_given? yield _conv(s) else ret_list << _conv(s) end end else if block_given? yield _conv(stat) else ret_list << _conv(stat) end end end ret_list if ! block_given? end
make directory @param [String] path
# File lib/hdfs_jruby.rb, line 167 def mkdir(path) @fs.mkdirs(_path(path)) end
@param [String] src hdfs source path @param [String] dst hdfs destination path
# File lib/hdfs_jruby.rb, line 138 def move(src, dst) @fs.rename(Path.new(src), Path.new(dst)) end
put file or directory to hdfs @param [String] local surouce (local path) @param [String] remote destination (hdfs path)
# File lib/hdfs_jruby.rb, line 174 def put(local, remote) @fs.copyFromLocalFile(Path.new(local), Path.new(remote)) end
set owner & group @param [String] path @param [String] owner @param [String] group
# File lib/hdfs_jruby.rb, line 211 def set_owner(path, owner, group) @fs.setOwner(_path(path), owner, group) end
set permission @param [String] path @param [Integer] perm permission
# File lib/hdfs_jruby.rb, line 203 def set_permission(path, perm) @fs.setPermission(_path(path), org.apache.hadoop.fs.permission.FsPermission.new(perm)) end
set working directory
# File lib/hdfs_jruby.rb, line 196 def set_working_directory(path) @fs.setWorkingDirectory(_path()) end
@return [Integer] file size
# File lib/hdfs_jruby.rb, line 161 def size(path) @fs.getFileStatus(_path(path)).getLen() end