class Dynamocli::Import
Constants
- ATTRIBUTE_TYPES_CONVERTERS
- BATCH_WRITE_ITEM_REQUEST_LIMIT
- LOGGER
- RANGE_TO_REMOVE_TYPE_FROM_HEADER
When the CSV comes from AWS, the header of the CSV is like this: “email (S)”. However, we cannot import the CSV with the header like this, we have to remove the part that specifies the type of the field before importing it.
- SUPPORTED_FILE_FORMATS
Attributes
dynamodb[R]
file[R]
table[R]
Public Class Methods
new(file:, table:, exported_from_aws: false)
click to toggle source
# File lib/dynamocli/import.rb, line 11 def initialize(file:, table:, exported_from_aws: false) @file = file @table = table @exported_from_aws = exported_from_aws @dynamodb = Aws::DynamoDB::Client.new end
Public Instance Methods
start()
click to toggle source
# File lib/dynamocli/import.rb, line 18 def start records = get_records write_records_to_dynamodb_table(records) LOGGER.success("#{records.size} record#{"s" if records.size != 1} imported to #{table}") rescue Aws::DynamoDB::Errors::ValidationException => e LOGGER.error(e.message) exit(42) end
Private Instance Methods
exported_from_aws?()
click to toggle source
# File lib/dynamocli/import.rb, line 54 def exported_from_aws? @exported_from_aws end
format_request_items(items)
click to toggle source
# File lib/dynamocli/import.rb, line 98 def format_request_items(items) { table => items.map { |item| { put_request: { item: item } } } } end
get_records()
click to toggle source
# File lib/dynamocli/import.rb, line 31 def get_records extension = File.extname(file) case extension when ".csv" records_from_csv(file) else LOGGER.error("Not supported file format. Only supported file formats are: #{SUPPORTED_FILE_FORMATS}") exit(42) end end
records_from_csv(csv)
click to toggle source
# File lib/dynamocli/import.rb, line 43 def records_from_csv(csv) set_custom_converter_for_csv csv_options = { encoding: "UTF-8", headers: true, converters: :attribute_definitions } records_csv = CSV.read(csv, csv_options) if exported_from_aws? transform_records_csv_from_aws(records_csv) else records_csv.map(&:to_hash) end end
set_custom_converter_for_csv()
click to toggle source
# File lib/dynamocli/import.rb, line 78 def set_custom_converter_for_csv attribute_definitions = dynamodb.describe_table(table_name: table).table.attribute_definitions CSV::Converters[:attribute_definitions] = lambda do |value, info| attribute_definition = attribute_definitions.find { |it| it.attribute_name == info.header } return value if attribute_definition.nil? ATTRIBUTE_TYPES_CONVERTERS[attribute_definition.attribute_type].call(value) end end
slice_items_to_attend_batch_write_limit(items)
click to toggle source
# File lib/dynamocli/import.rb, line 94 def slice_items_to_attend_batch_write_limit(items) items.each_slice(BATCH_WRITE_ITEM_REQUEST_LIMIT) end
transform_records_csv_from_aws(records_csv)
click to toggle source
# File lib/dynamocli/import.rb, line 63 def transform_records_csv_from_aws(records_csv) records_csv.map do |record_csv| record = record_csv.to_h record.each_with_object({}) do |(key, value), records| records[key[RANGE_TO_REMOVE_TYPE_FROM_HEADER]] = value end end end
write_records_to_dynamodb_table(records)
click to toggle source
# File lib/dynamocli/import.rb, line 87 def write_records_to_dynamodb_table(records) slice_items_to_attend_batch_write_limit(records).each do |items| dynamodb.batch_write_item(request_items: format_request_items(items)) end end