class BeetleETL::Import
Public Class Methods
new(config)
click to toggle source
# File lib/beetle_etl/import.rb, line 4 def initialize(config) @config = config @report = {} @transformations ||= TransformationLoader.new(@config).load end
Public Instance Methods
merge_report!(new_report)
click to toggle source
# File lib/beetle_etl/import.rb, line 63 def merge_report!(new_report) @report.merge!(new_report) { |key, oldval, newval| oldval.merge(newval) } end
run()
click to toggle source
# File lib/beetle_etl/import.rb, line 11 def run begin run_setup run_transform run_load ensure run_cleanup end @report end
run_cleanup()
click to toggle source
# File lib/beetle_etl/import.rb, line 55 def run_cleanup steps = @transformations.map { |t| DropStage.new(@config, t.table_name) } merge_report! StepRunner.new(@config, steps).run end
run_load()
click to toggle source
# File lib/beetle_etl/import.rb, line 43 def run_load steps = @transformations.map { |t| Load.new(@config, t.table_name, t.relations) } result = @config.database.transaction do StepRunner.new(@config, steps).run end merge_report! result end
run_setup()
click to toggle source
# File lib/beetle_etl/import.rb, line 23 def run_setup steps = @transformations.map { |t| CreateStage.new(@config, t.table_name, t.relations, t.column_names) } merge_report! StepRunner.new(@config, steps).run end
run_transform()
click to toggle source
# File lib/beetle_etl/import.rb, line 31 def run_transform steps = @transformations.flat_map { |t| [ Transform.new(@config, t.table_name, t.dependencies, t.query), MapRelations.new(@config, t.table_name, t.relations), TableDiff.new(@config, t.table_name) ] } merge_report! AsyncStepRunner.new(@config, steps).run end