class RedshiftConnector::S3DataFileBundle
Attributes
bucket[R]
logger[R]
prefix[R]
Public Class Methods
for_params(params)
click to toggle source
# File lib/redshift_connector/s3_data_file_bundle.rb, line 9 def self.for_params(params) unless params.txn_id raise ArgumentError, "cannot create bundle: missing txn_id" end s3bucket = params.bucket ? S3Bucket.get(params.bucket) : S3Bucket.default for_table( bucket: s3bucket, schema: params.schema, table: params.table, txn_id: params.txn_id, logger: params.logger ) end
for_prefix(bucket: S3Bucket.default, prefix:, format:, logger: RedshiftConnector.logger)
click to toggle source
# File lib/redshift_connector/s3_data_file_bundle.rb, line 23 def self.for_prefix(bucket: S3Bucket.default, prefix:, format:, logger: RedshiftConnector.logger) real_prefix = "#{bucket.prefix}/#{prefix}" new(bucket, real_prefix, format: format, logger: logger) end
for_table(bucket: S3Bucket.default, schema:, table:, txn_id:, logger: RedshiftConnector.logger)
click to toggle source
# File lib/redshift_connector/s3_data_file_bundle.rb, line 28 def self.for_table(bucket: S3Bucket.default, schema:, table:, txn_id:, logger: RedshiftConnector.logger) prefix = "#{bucket.prefix}/#{schema}_export/#{table}/#{txn_id}/#{table}.csv." new(bucket, prefix, format: :redshift_csv, logger: logger) end
new(bucket, prefix, format: :csv, logger: RedshiftConnector.logger)
click to toggle source
# File lib/redshift_connector/s3_data_file_bundle.rb, line 33 def initialize(bucket, prefix, format: :csv, logger: RedshiftConnector.logger) @bucket = bucket @prefix = prefix @format = format @logger = logger @reader_class = Reader.get(format) logger.info "reader: #{@reader_class}" end
Public Instance Methods
clear()
click to toggle source
# File lib/redshift_connector/s3_data_file_bundle.rb, line 59 def clear pref = File.dirname(@prefix) + '/' keys = @bucket.objects(prefix: pref).map(&:key) unless keys.empty? logger.info "DELETE #{pref}*" @bucket.delete_objects(keys) end end
credential_string()
click to toggle source
# File lib/redshift_connector/s3_data_file_bundle.rb, line 50 def credential_string @bucket.credential_string end
data_files()
click to toggle source
# File lib/redshift_connector/s3_data_file_bundle.rb, line 54 def data_files @bucket.objects(prefix: @prefix) .map {|obj| S3DataFile.new(obj, reader_class: @reader_class) } end
url()
click to toggle source
# File lib/redshift_connector/s3_data_file_bundle.rb, line 46 def url "s3://#{@bucket.name}/#{@prefix}" end