class ExportToGcloud::Exporter

Public Class Methods

define(**kwargs, &block) click to toggle source
# File lib/export_to_gcloud/exporters/exporter.rb, line 87
def self.define **kwargs, &block
  ::ExportToGcloud::Exporter::Definition.set_last_definition self, kwargs, &block
end
new(definition, context) click to toggle source
# File lib/export_to_gcloud/exporters/exporter.rb, line 6
def initialize definition, context
  @definition = definition
  @context = context

  @parts = []
  case definition.parts
    when Array then definition.parts.each{|label, *part_args| add_data_part *part_args, label: label}
    when Proc  then definition.parts.call self
  end
end

Public Instance Methods

add_data_part(*args, label:nil) click to toggle source
# File lib/export_to_gcloud/exporters/exporter.rb, line 26
def add_data_part *args, label:nil
  args.unshift(label ? label.to_s : (@parts.length+1).to_s)
  @parts << args
end
bq_table() click to toggle source
# File lib/export_to_gcloud/exporters/exporter.rb, line 62
def bq_table
  unless defined? @bq_table
    @bq_table = @context.dataset.table @definition.get_bq_table_name
  end
  @bq_table
end
get_storage_files() click to toggle source
# File lib/export_to_gcloud/exporters/exporter.rb, line 56
def get_storage_files
  @parts.map do |label, *_|
    @context.bucket.file storage_file_path(label)
  end.compact
end
local_file_path(label) click to toggle source
# File lib/export_to_gcloud/exporters/exporter.rb, line 17
def local_file_path label
  @context.dump_path.join "#{@definition.name}#{prepend_underscore label}.csv"
end
process_all_parts!(recreate_table=true) click to toggle source
# File lib/export_to_gcloud/exporters/exporter.rb, line 31
def process_all_parts! recreate_table=true
  add_data_part label: 'all' if @parts.empty?
  recreate_bq_table! if recreate_table

  @parts.map{|args| process_part! *args}
end
process_part!(label, *part_args) click to toggle source
# File lib/export_to_gcloud/exporters/exporter.rb, line 38
def process_part! label, *part_args
  file = local_file_path label
  create_data_file! file, *part_args

  storage_name = storage_file_path label
  gcloud_file = upload_file! file, storage_name
  start_load_job gcloud_file
end
recreate_bq_table!() click to toggle source
# File lib/export_to_gcloud/exporters/exporter.rb, line 69
def recreate_bq_table!
  bq_table.delete if bq_table
  @bq_table = @context.dataset.create_table @definition.get_bq_table_name, &@definition.bq_schema
end
start_load_job(gcloud_file, **_load_settings) click to toggle source
# File lib/export_to_gcloud/exporters/exporter.rb, line 74
def start_load_job gcloud_file, **_load_settings
  load_settings = {
      format: 'csv',
      quote: '"',
      delimiter: ';',
      create: 'never',
      write: 'append',
      max_bad_records: 0
  }
  load_settings.merge! _load_settings unless _load_settings.empty?
  bq_table.load gcloud_file, **load_settings
end
storage_file_path(label) click to toggle source
# File lib/export_to_gcloud/exporters/exporter.rb, line 21
def storage_file_path label
  prefix = @definition.storage_prefix || @context.storage_prefix
  "#{prefix}#{@definition.name}#{prepend_underscore label}.csv"
end
upload_file!(file, storage_name) click to toggle source
# File lib/export_to_gcloud/exporters/exporter.rb, line 49
def upload_file!(file, storage_name)
  file = compress_file! file
  gcloud_file = @context.bucket.create_file file, storage_name, chunk_size: 2**21 # 2MB
  file.delete
  gcloud_file
end

Private Instance Methods

compress_file!(original_file) click to toggle source
# File lib/export_to_gcloud/exporters/exporter.rb, line 97
def compress_file!(original_file)
  err = %x(pigz -f9 #{original_file.to_path} 2>&1)
  compressed_file = Pathname.new "#{original_file.to_path}.gz"
  raise "Compression of #{original_file.to_path} failed: #{err}" unless compressed_file.exist?
  original_file.delete if original_file.exist?
  compressed_file
end
create_data_file!(file, *part_data) click to toggle source
# File lib/export_to_gcloud/exporters/exporter.rb, line 93
def create_data_file! file, *part_data
  File.write file.to_path, @definition.get_data(*part_data)
end
prepend_underscore(text) click to toggle source
# File lib/export_to_gcloud/exporters/exporter.rb, line 105
def prepend_underscore text
  "_#{text}" if String === text && !text.empty?
end