class Sink3::PathCrawler

Public Class Methods

new(path, prefix, parent=nil) click to toggle source
# File lib/sink3/path_crawler.rb, line 5
def initialize(path, prefix, parent=nil)
  @parent = parent
  @prefix = prefix

  if @parent.nil? 
    @path = Pathname.new(path)
  else
    @path = parent.join(path)
  end

  # If path is '.' then just skip the prefix
  if path == prefix 
    @prefix = nil 
  end
end

Public Instance Methods

start() click to toggle source
# File lib/sink3/path_crawler.rb, line 21
def start
  raise "Path does not exist" unless @path.exist? 
  if @path.directory? 
    @path.opendir.each do |file| 
      next if file.start_with? '.'
      PathCrawler.new(file, @prefix, @path).start
    end
  else
    send_to_remote
  end
end

Private Instance Methods

bucket() click to toggle source
# File lib/sink3/path_crawler.rb, line 43
def bucket 
  s3.buckets[ENV['BUCKET']]
end
check_exists(remote_file) click to toggle source
# File lib/sink3/path_crawler.rb, line 47
def check_exists remote_file
  remote_file.exists?
rescue AWS::S3::Errors::Forbidden
  # expect this error to be returned when file exists
  # and we don't have access to read the file
  true 
end
formatted_date() click to toggle source
# File lib/sink3/path_crawler.rb, line 74
def formatted_date
  if !Sink3.config.skip_date_partition
    return "#{Time.now.strftime "%Y-%m-%d"}/"
  else
    return ""
  end
end
prefix_and_path() click to toggle source
# File lib/sink3/path_crawler.rb, line 82
def prefix_and_path
  if @prefix.nil? 
    "#{@path}" 
  else
    "#{@prefix}/#{@path}"
  end
end
s3() click to toggle source
# File lib/sink3/path_crawler.rb, line 35
def s3
  @s3 ||= AWS::S3.new(
    :access_key_id => ENV['ACCESS_KEY'],
    :secret_access_key => ENV['SECRET_KEY'], 
    :region => ENV['REGION']
  )
end
send_to_remote() click to toggle source
# File lib/sink3/path_crawler.rb, line 55
def send_to_remote
  remote_path = "#{ENV['HOSTNAME'].strip}/#{formatted_date}#{prefix_and_path}"
  remote_file = bucket.objects[remote_path]
  if Sink3.config.skip_overwrite 
    if ! check_exists(remote_file)
      puts "remote file does not exist #{remote_path}"  if Sink3.config.verbose
      remote_file.write(@path)
    else
      puts "skpping overwrite of #{remote_path}"  if Sink3.config.verbose
    end
  else
    remote_file.write(@path)
  end

  if Sink3.config.delete_after_upload? 
    FileUtils.rm @path
  end
end