class RedshiftConnector::S3DataFileBundle

Attributes

bucket[R]
logger[R]
prefix[R]

Public Class Methods

for_params(params) click to toggle source
# File lib/redshift_connector/s3_data_file_bundle.rb, line 9
def self.for_params(params)
  unless params.txn_id
    raise ArgumentError, "cannot create bundle: missing txn_id"
  end
  s3bucket = params.bucket ? S3Bucket.get(params.bucket) : S3Bucket.default
  for_table(
    bucket: s3bucket,
    schema: params.schema,
    table: params.table,
    txn_id: params.txn_id,
    logger: params.logger
  )
end
for_prefix(bucket: S3Bucket.default, prefix:, format:, logger: RedshiftConnector.logger) click to toggle source
# File lib/redshift_connector/s3_data_file_bundle.rb, line 23
def self.for_prefix(bucket: S3Bucket.default, prefix:, format:, logger: RedshiftConnector.logger)
  real_prefix = "#{bucket.prefix}/#{prefix}"
  new(bucket, real_prefix, format: format, logger: logger)
end
for_table(bucket: S3Bucket.default, schema:, table:, txn_id:, logger: RedshiftConnector.logger) click to toggle source
# File lib/redshift_connector/s3_data_file_bundle.rb, line 28
def self.for_table(bucket: S3Bucket.default, schema:, table:, txn_id:, logger: RedshiftConnector.logger)
  prefix = "#{bucket.prefix}/#{schema}_export/#{table}/#{txn_id}/#{table}.csv."
  new(bucket, prefix, format: :redshift_csv, logger: logger)
end
new(bucket, prefix, format: :csv, logger: RedshiftConnector.logger) click to toggle source
# File lib/redshift_connector/s3_data_file_bundle.rb, line 33
def initialize(bucket, prefix, format: :csv, logger: RedshiftConnector.logger)
  @bucket = bucket
  @prefix = prefix
  @format = format
  @logger = logger
  @reader_class = Reader.get(format)
  logger.info "reader: #{@reader_class}"
end

Public Instance Methods

clear() click to toggle source
# File lib/redshift_connector/s3_data_file_bundle.rb, line 59
def clear
  pref = File.dirname(@prefix) + '/'
  keys = @bucket.objects(prefix: pref).map(&:key)
  unless keys.empty?
    logger.info "DELETE #{pref}*"
    @bucket.delete_objects(keys)
  end
end
credential_string() click to toggle source
# File lib/redshift_connector/s3_data_file_bundle.rb, line 50
def credential_string
  @bucket.credential_string
end
data_files() click to toggle source
# File lib/redshift_connector/s3_data_file_bundle.rb, line 54
def data_files
  @bucket.objects(prefix: @prefix)
    .map {|obj| S3DataFile.new(obj, reader_class: @reader_class) }
end
url() click to toggle source
# File lib/redshift_connector/s3_data_file_bundle.rb, line 46
def url
  "s3://#{@bucket.name}/#{@prefix}"
end