module Hdfs

Constants

HADOOP_HOME
JAR_PATTERN_0_20
VERSION

Public Class Methods

_conv(stat) click to toggle source

@private

# File lib/hdfs_jruby.rb, line 249
def _conv(stat)
  file_info = {}
  file_info['path'] = stat.getPath.to_s
  file_info['length'] = stat.getLen.to_i
  file_info['modificationTime'] = stat.getModificationTime.to_i
  file_info['owner'] = stat.getOwner.to_s
  file_info['group'] = stat.getGroup.to_s
  file_info['permission'] = stat.getPermission.toShort.to_i
  file_info['type'] = !stat.isDir ? 'FILE': 'DIRECTORY'
  return file_info
end

_path(path) click to toggle source

@private

# File lib/hdfs_jruby.rb, line 241
def _path(path)
  if path.nil?
    raise "path is nil"
  end
  Path.new(path)
end

connectAsUser(user) click to toggle source

@private

# File lib/hdfs_jruby.rb, line 54
def connectAsUser(user)
  uri =  Hdfs::FileSystem.getDefaultUri(@conf)
  @fs.close if ! @fs.nil?
  @fs = Hdfs::FileSystem.get(uri, @conf, user)
end

delete(path, r=false) click to toggle source

delete

@param [String] path @param [Boolean] r recursive false or true (default: false)

# File lib/hdfs_jruby.rb, line 146
def delete(path, r=false)
  @fs.delete(_path(path), r)
end

directory?(path) click to toggle source

@return [Boolean] true: directory, false: file

# File lib/hdfs_jruby.rb, line 156
def directory?(path)
  @fs.isDirectory(_path(path))
end

exists?(path) click to toggle source

@param [String] path

# File lib/hdfs_jruby.rb, line 132
def exists?(path)
  @fs.exists(_path(path))
end

file?(path) click to toggle source

@return [Boolean] true: file, false: directory

# File lib/hdfs_jruby.rb, line 151
def file?(path)
  @fs.isFile(_path(path))
end

get(remote, local) click to toggle source

get file or directory from hdfs @param [String] remote surouce (hdfs path) @param [String] local destination (local path)

# File lib/hdfs_jruby.rb, line 181
def get(remote, local)
  @fs.copyToLocalFile(Path.new(remote), Path.new(local))
end

get_fs() click to toggle source

# File lib/hdfs_jruby.rb, line 215
def get_fs
  @fs
end

get_home_directory() click to toggle source

get home directory

# File lib/hdfs_jruby.rb, line 186
def get_home_directory()
  @fs.getHomeDirectory()
end

get_working_directory() click to toggle source

get working directory

# File lib/hdfs_jruby.rb, line 191
def get_working_directory()
  @fs.getWorkingDirectory()
end

list(path, opts={}) { |_conv(stat)| ... } click to toggle source

@private

# File lib/hdfs_jruby.rb, line 106
def list(path, opts={})
  use_glob = opts[:glob] ? true : false
  p = _path(path)

  list = nil
  if use_glob
    list = @fs.globStatus(p)
  else
    list = @fs.listStatus(p)
  end
  return [] if list.nil?
    
  if ! block_given?
    ret_list = []
    list.each do | stat |
      ret_list << _conv(stat)
    end
    return ret_list
  else
    list.each do | stat |
      yield _conv(stat)
    end
  end
end

ls(path) { |_conv(s)| ... } click to toggle source

ls @example

Hdfs.ls("hoge/").each do | stat |
  p stat
end

@param [String] path @return [Array] file status array

@note file status:

path
length
modificationTime
owner
group
permission
type

# File lib/hdfs_jruby.rb, line 76
def ls(path)
  p = _path(path)
  list = @fs.globStatus(p)
  return [] if list.nil?

  ret_list = []
  list.each do |stat|
    if stat.isDir
      sub_list = @fs.listStatus(stat.getPath)
      next if sub_list.nil?
      
      sub_list.each do | s |
        if block_given?
          yield _conv(s)
        else
          ret_list << _conv(s)
        end
      end
    else
      if block_given?
        yield _conv(stat)
      else
        ret_list << _conv(stat)
      end
    end
  end
  ret_list if ! block_given?
end

mkdir(path) click to toggle source

make directory @param [String] path

# File lib/hdfs_jruby.rb, line 167
def mkdir(path)
  @fs.mkdirs(_path(path))
end

move(src, dst) click to toggle source

@param [String] src hdfs source path @param [String] dst hdfs destination path

# File lib/hdfs_jruby.rb, line 138
def move(src, dst)
  @fs.rename(Path.new(src), Path.new(dst))
end

put(local, remote) click to toggle source

put file or directory to hdfs @param [String] local surouce (local path) @param [String] remote destination (hdfs path)

# File lib/hdfs_jruby.rb, line 174
def put(local, remote)
  @fs.copyFromLocalFile(Path.new(local), Path.new(remote))
end

set_owner(path, owner, group) click to toggle source

set owner & group @param [String] path @param [String] owner @param [String] group

# File lib/hdfs_jruby.rb, line 211
def set_owner(path, owner, group)
  @fs.setOwner(_path(path), owner, group)
end

set_permission(path, perm) click to toggle source

set permission @param [String] path @param [Integer] perm permission

# File lib/hdfs_jruby.rb, line 203
def set_permission(path, perm)
  @fs.setPermission(_path(path), org.apache.hadoop.fs.permission.FsPermission.new(perm))
end

set_working_directory(path) click to toggle source

set working directory

# File lib/hdfs_jruby.rb, line 196
def set_working_directory(path)
  @fs.setWorkingDirectory(_path())
end

size(path) click to toggle source

@return [Integer] file size

# File lib/hdfs_jruby.rb, line 161
def size(path)
  @fs.getFileStatus(_path(path)).getLen()
end

Private Instance Methods

_conv(stat) click to toggle source

@private

# File lib/hdfs_jruby.rb, line 249
def _conv(stat)
  file_info = {}
  file_info['path'] = stat.getPath.to_s
  file_info['length'] = stat.getLen.to_i
  file_info['modificationTime'] = stat.getModificationTime.to_i
  file_info['owner'] = stat.getOwner.to_s
  file_info['group'] = stat.getGroup.to_s
  file_info['permission'] = stat.getPermission.toShort.to_i
  file_info['type'] = !stat.isDir ? 'FILE': 'DIRECTORY'
  return file_info
end

_path(path) click to toggle source

@private

# File lib/hdfs_jruby.rb, line 241
def _path(path)
  if path.nil?
    raise "path is nil"
  end
  Path.new(path)
end

connectAsUser(user) click to toggle source

@private

# File lib/hdfs_jruby.rb, line 54
def connectAsUser(user)
  uri =  Hdfs::FileSystem.getDefaultUri(@conf)
  @fs.close if ! @fs.nil?
  @fs = Hdfs::FileSystem.get(uri, @conf, user)
end

delete(path, r=false) click to toggle source

delete

@param [String] path @param [Boolean] r recursive false or true (default: false)

# File lib/hdfs_jruby.rb, line 146
def delete(path, r=false)
  @fs.delete(_path(path), r)
end

directory?(path) click to toggle source

@return [Boolean] true: directory, false: file

# File lib/hdfs_jruby.rb, line 156
def directory?(path)
  @fs.isDirectory(_path(path))
end

exists?(path) click to toggle source

@param [String] path

# File lib/hdfs_jruby.rb, line 132
def exists?(path)
  @fs.exists(_path(path))
end

file?(path) click to toggle source

@return [Boolean] true: file, false: directory

# File lib/hdfs_jruby.rb, line 151
def file?(path)
  @fs.isFile(_path(path))
end

get(remote, local) click to toggle source

get file or directory from hdfs @param [String] remote surouce (hdfs path) @param [String] local destination (local path)

# File lib/hdfs_jruby.rb, line 181
def get(remote, local)
  @fs.copyToLocalFile(Path.new(remote), Path.new(local))
end

get_fs() click to toggle source

# File lib/hdfs_jruby.rb, line 215
def get_fs
  @fs
end

get_home_directory() click to toggle source

get home directory

# File lib/hdfs_jruby.rb, line 186
def get_home_directory()
  @fs.getHomeDirectory()
end

get_working_directory() click to toggle source

get working directory

# File lib/hdfs_jruby.rb, line 191
def get_working_directory()
  @fs.getWorkingDirectory()
end

list(path, opts={}) { |_conv(stat)| ... } click to toggle source

@private

# File lib/hdfs_jruby.rb, line 106
def list(path, opts={})
  use_glob = opts[:glob] ? true : false
  p = _path(path)

  list = nil
  if use_glob
    list = @fs.globStatus(p)
  else
    list = @fs.listStatus(p)
  end
  return [] if list.nil?
    
  if ! block_given?
    ret_list = []
    list.each do | stat |
      ret_list << _conv(stat)
    end
    return ret_list
  else
    list.each do | stat |
      yield _conv(stat)
    end
  end
end

ls(path) { |_conv(s)| ... } click to toggle source

ls @example

Hdfs.ls("hoge/").each do | stat |
  p stat
end

@param [String] path @return [Array] file status array

@note file status:

path
length
modificationTime
owner
group
permission
type

# File lib/hdfs_jruby.rb, line 76
def ls(path)
  p = _path(path)
  list = @fs.globStatus(p)
  return [] if list.nil?

  ret_list = []
  list.each do |stat|
    if stat.isDir
      sub_list = @fs.listStatus(stat.getPath)
      next if sub_list.nil?
      
      sub_list.each do | s |
        if block_given?
          yield _conv(s)
        else
          ret_list << _conv(s)
        end
      end
    else
      if block_given?
        yield _conv(stat)
      else
        ret_list << _conv(stat)
      end
    end
  end
  ret_list if ! block_given?
end

mkdir(path) click to toggle source

make directory @param [String] path

# File lib/hdfs_jruby.rb, line 167
def mkdir(path)
  @fs.mkdirs(_path(path))
end

move(src, dst) click to toggle source

@param [String] src hdfs source path @param [String] dst hdfs destination path

# File lib/hdfs_jruby.rb, line 138
def move(src, dst)
  @fs.rename(Path.new(src), Path.new(dst))
end

put(local, remote) click to toggle source

put file or directory to hdfs @param [String] local surouce (local path) @param [String] remote destination (hdfs path)

# File lib/hdfs_jruby.rb, line 174
def put(local, remote)
  @fs.copyFromLocalFile(Path.new(local), Path.new(remote))
end

set_owner(path, owner, group) click to toggle source

set owner & group @param [String] path @param [String] owner @param [String] group

# File lib/hdfs_jruby.rb, line 211
def set_owner(path, owner, group)
  @fs.setOwner(_path(path), owner, group)
end

set_permission(path, perm) click to toggle source

set permission @param [String] path @param [Integer] perm permission

# File lib/hdfs_jruby.rb, line 203
def set_permission(path, perm)
  @fs.setPermission(_path(path), org.apache.hadoop.fs.permission.FsPermission.new(perm))
end

set_working_directory(path) click to toggle source

set working directory

# File lib/hdfs_jruby.rb, line 196
def set_working_directory(path)
  @fs.setWorkingDirectory(_path())
end

size(path) click to toggle source

@return [Integer] file size

# File lib/hdfs_jruby.rb, line 161
def size(path)
  @fs.getFileStatus(_path(path)).getLen()
end