class BeetleETL::Import

Public Class Methods

new(config) click to toggle source
# File lib/beetle_etl/import.rb, line 4
def initialize(config)
  @config = config
  @report = {}

  @transformations ||= TransformationLoader.new(@config).load
end

Public Instance Methods

merge_report!(new_report) click to toggle source
# File lib/beetle_etl/import.rb, line 63
def merge_report!(new_report)
  @report.merge!(new_report) { |key, oldval, newval| oldval.merge(newval) }
end
run() click to toggle source
# File lib/beetle_etl/import.rb, line 11
def run
  begin
    run_setup
    run_transform
    run_load
  ensure
    run_cleanup
  end

  @report
end
run_cleanup() click to toggle source
# File lib/beetle_etl/import.rb, line 55
def run_cleanup
  steps = @transformations.map { |t|
    DropStage.new(@config, t.table_name)
  }

  merge_report! StepRunner.new(@config, steps).run
end
run_load() click to toggle source
# File lib/beetle_etl/import.rb, line 43
def run_load
  steps = @transformations.map { |t|
    Load.new(@config, t.table_name, t.relations)
  }

  result = @config.database.transaction do
    StepRunner.new(@config, steps).run
  end

  merge_report! result
end
run_setup() click to toggle source
# File lib/beetle_etl/import.rb, line 23
def run_setup
  steps = @transformations.map { |t|
    CreateStage.new(@config, t.table_name, t.relations, t.column_names)
  }

  merge_report! StepRunner.new(@config, steps).run
end
run_transform() click to toggle source
# File lib/beetle_etl/import.rb, line 31
def run_transform
  steps = @transformations.flat_map { |t|
    [
      Transform.new(@config, t.table_name, t.dependencies, t.query),
      MapRelations.new(@config, t.table_name, t.relations),
      TableDiff.new(@config, t.table_name)
    ]
  }

  merge_report! AsyncStepRunner.new(@config, steps).run
end