class Embulk::Filter::Unique

Public Class Methods

transaction(config, in_schema) { |task, in_schema| ... } click to toggle source
# File lib/embulk/filter/unique.rb, line 9
def self.transaction(config, in_schema, &control)
  task = {
    "columns" => config.param("columns", :array),
  }

  yield(task, in_schema)
end

Public Instance Methods

add(page) click to toggle source
# File lib/embulk/filter/unique.rb, line 25
def add(page)
  page.each do |record|
    part = {}
    @cols.each do |c|
      cs = page.schema.select{|s| s.name == c}
      idx = cs[0].index
      part[c] = record[idx]
    end

    unless @exists.include?(part)
      page_builder.add(record)
      @exists.add(part)
    end
  end
end
close() click to toggle source
# File lib/embulk/filter/unique.rb, line 22
def close
end
finish() click to toggle source
# File lib/embulk/filter/unique.rb, line 41
def finish
  page_builder.finish
end
init() click to toggle source
# File lib/embulk/filter/unique.rb, line 17
def init
  @cols = task["columns"]
  @exists = Set.new
end