class Sink3::PathCrawler
Public Class Methods
new(path, prefix, parent=nil)
click to toggle source
# File lib/sink3/path_crawler.rb, line 5 def initialize(path, prefix, parent=nil) @parent = parent @prefix = prefix if @parent.nil? @path = Pathname.new(path) else @path = parent.join(path) end # If path is '.' then just skip the prefix if path == prefix @prefix = nil end end
Public Instance Methods
start()
click to toggle source
# File lib/sink3/path_crawler.rb, line 21 def start raise "Path does not exist" unless @path.exist? if @path.directory? @path.opendir.each do |file| next if file.start_with? '.' PathCrawler.new(file, @prefix, @path).start end else send_to_remote end end
Private Instance Methods
bucket()
click to toggle source
# File lib/sink3/path_crawler.rb, line 43 def bucket s3.buckets[ENV['BUCKET']] end
check_exists(remote_file)
click to toggle source
# File lib/sink3/path_crawler.rb, line 47 def check_exists remote_file remote_file.exists? rescue AWS::S3::Errors::Forbidden # expect this error to be returned when file exists # and we don't have access to read the file true end
formatted_date()
click to toggle source
# File lib/sink3/path_crawler.rb, line 74 def formatted_date if !Sink3.config.skip_date_partition return "#{Time.now.strftime "%Y-%m-%d"}/" else return "" end end
prefix_and_path()
click to toggle source
# File lib/sink3/path_crawler.rb, line 82 def prefix_and_path if @prefix.nil? "#{@path}" else "#{@prefix}/#{@path}" end end
s3()
click to toggle source
# File lib/sink3/path_crawler.rb, line 35 def s3 @s3 ||= AWS::S3.new( :access_key_id => ENV['ACCESS_KEY'], :secret_access_key => ENV['SECRET_KEY'], :region => ENV['REGION'] ) end
send_to_remote()
click to toggle source
# File lib/sink3/path_crawler.rb, line 55 def send_to_remote remote_path = "#{ENV['HOSTNAME'].strip}/#{formatted_date}#{prefix_and_path}" remote_file = bucket.objects[remote_path] if Sink3.config.skip_overwrite if ! check_exists(remote_file) puts "remote file does not exist #{remote_path}" if Sink3.config.verbose remote_file.write(@path) else puts "skpping overwrite of #{remote_path}" if Sink3.config.verbose end else remote_file.write(@path) end if Sink3.config.delete_after_upload? FileUtils.rm @path end end