class DataListConverter

this class is used for convert data between types, so we can use it to easily export and import data.

item_data: [{name: ‘xx’, value: 12}, …] table_data: [[‘name’, ‘value’], [‘xx’, ‘12’], ..] item_iterator and table_iterator are iterators which yield each row

usage: require ‘data_list_converter/types/fast_xlsx_file’ d = [{a: 12.3, b: ‘xx’}] DataListConverter.save_to_file(“t.xls”, d, :item_data, file_format: :fast_xlsx_file)

multi_sheet_table_data = {

'sheet1' => [columns, row, row, ...],
'sheet2' => [columns, row, row, ...],

} also: multi_sheet_item_data multi_sheet_table_iterator multi_sheet_item_iterator

ActiveRecords

Constants

CONVERTERS
FILTERS
MULT_SHEET_CONVERTS
VERSION

Attributes

debug[RW]

Public Class Methods

convert(from_type, to_type, from_value, options={}) click to toggle source

Example: convert(:item_iterator, :item_data, iter) convert(:item_iterator, :csv_file, iter, csv_file: {filename: ‘result.csv’}) convert(:csv_file, :item_data, {filename: ‘result.csv’})

can add filter: filter = :limit filter = {limit: {size: 2}} filter = [{limit: {size: 12}}, {count: {size: 4}}] convert(:item_iterator, :table_data, iter, table_iterator: {filter: filter})

# File lib/data_list_converter/base.rb, line 45
def convert(from_type, to_type, from_value, options={})
  methods = []
  add_filter = lambda { |type|
    filters = (options[type] || {})[:filter]
    return unless filters
    methods += normalize_filters(type, filters)
  }

  route = find_route(from_type, to_type)
  add_filter.call(route[0])

  self.log("route: #{route}")
  (0..(route.length-2)).map do |i|
    from_type, to_type = route[i], route[i+1]
    method = CONVERTERS[[from_type, to_type]]
    raise "cannot find converter #{from_type} -> #{to_type}" unless method
    self.log "#{from_type} -> #{to_type} options: #{options[to_type]}"
    methods.push([method, options[to_type] || {}])
    add_filter.call(to_type)
  end

  self.log("methods: #{methods}")
  methods.inject(from_value) do |v, method|
    method, args = method
    method.call(v, args)
  end
end
data_to_iterator(data, options={}) click to toggle source
# File lib/data_list_converter/types/basic.rb, line 40
def self.data_to_iterator(data, options={})
  lambda { |&block|
    data.each do |d|
      block.call(d)
    end
  }
end
file_types() click to toggle source
# File lib/data_list_converter/helper.rb, line 31
def self.file_types
  matcher = /(.*)_file$/
  DataListConverter.types.select do |type|
    matcher.match(type)
  end
end
find_route(from_type, to_type) click to toggle source

One type of data can be converted into any other types, we have a list of convert methods: CONVERTERS If we want to convert between types, like: convert item_data into csv_file, we need find all the intermidate data type, like: [:item_data, :item_iterator, :table_iterator, :csv_file]

# File lib/data_list_converter/base.rb, line 95
def find_route(from_type, to_type)
  [from_type, to_type].each do |type|
    raise Exception, "cannot find type: #{type}" unless self.types.include?(type)
  end
  raise Exception, "from_type should not equal to to_type: #{from_type}" if from_type == to_type

  # map wide search
  checked = Set.new
  checking = Set.new([from_type])
  directions = {}

  while not checking.empty?
    current_node = checking.first

    next_nodes = route_map[current_node]
    # mark direction from from_type
    next_nodes.each do |node|
      # first marked is the shortest
      directions[node] ||= current_node
    end

    if next_nodes.include?(to_type)
      # get route
      start = to_type
      route = [start]
      while start != from_type
        previous = directions[start]
        raise "cannot find previous for #{start} in #{directions}" if not previous
        route.push(previous)
        start = previous
      end
      return route.reverse
    else
      checking.delete(current_node)
      checked.add(current_node)
      checking += Set.new(next_nodes) - checked
    end
  end

  log = ["Route not found: #{from_type} -> #{to_type}", "Current routes:"]
  log += self.routes.map{|from, to| "#{from} -> #{to}"}
  raise Exception, log.join("\n")
end
flatten(data, sep=':', max_level=nil) click to toggle source

flatten multi level item data into one level, example:

{a: {b: 12}, c: {d: {e: 11}}}
=>
{:"a:b"=>12, :"c:d:e"=>11}
# File lib/data_list_converter/helper.rb, line 59
def self.flatten(data, sep=':', max_level=nil)
  out = {}
  recursive_flatten(out, data, nil, sep, 1, max_level)
  out
end
get_file_format(filename) click to toggle source
# File lib/data_list_converter/helper.rb, line 15
def self.get_file_format(filename)
  file_type = (File.extname(filename)[1..-1] + "_file").to_sym
  unless DataListConverter.file_types.include?(file_type)
    raise "unknown file format: #{file_type}"
  end
  file_type
end
iterator_limit(proc, options) click to toggle source
# File lib/data_list_converter/filters/limit.rb, line 2
def self.iterator_limit(proc, options)
  limit_size = options[:size] || 10
  lambda { |&block|
    limit = 0
    proc.call do |item|
      block.call(item)
      limit += 1
      break if limit >= limit_size
    end
  }
end
iterator_to_data(proc, options={}) click to toggle source
# File lib/data_list_converter/types/basic.rb, line 32
def self.iterator_to_data(proc, options={})
  out = []
  proc.call { |d| out << d }
  out
end
load_from_file(filename, data_format=:item_data, options={}) click to toggle source
# File lib/data_list_converter/helper.rb, line 9
def self.load_from_file(filename, data_format=:item_data, options={})
  file_format = options.delete(:file_format) || self.get_file_format(filename)
  options[:filename] = filename
  DataListConverter.convert(file_format, data_format, options)
end
log(msg) click to toggle source
# File lib/data_list_converter/base.rb, line 19
def log(msg)
  return unless debug
  puts "#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}\t#{msg}"
end
marshal_data_to_file(data, options) click to toggle source
# File lib/data_list_converter/types/marshal.rb, line 10
def self.marshal_data_to_file(data, options)
  filename = self.parameter(options, :filename, :marshal)
  File.open(filename, 'w+') do |f|
    Marshal.dump(data, f)
  end
  options[:filename]
end
marshal_file_to_data(input, options=nil) click to toggle source
# File lib/data_list_converter/types/marshal.rb, line 3
def self.marshal_file_to_data(input, options=nil)
  filename = self.parameter(input, :filename, :input)
  File.open(filename) do |f|
    Marshal.load(f)
  end
end
normalize_filters(type, filters) click to toggle source
# File lib/data_list_converter/base.rb, line 73
def normalize_filters(type, filters)
  # filter list as array
  filters = [filters] unless filters.kind_of?(Array)
  filters.map do |v|
    # fix filter arguments
    case v
    # {:limit, {count: 12}} => [:limit, {count: 12}]
    when Hash; v.first
    # :debug => [:debug, {}]
    when Symbol, String; [v, {}]
    else; v
    end
  end.map do |name, args|
    method = FILTERS[type][name] rescue raise("cannot find method for type #{type} filter #{name}")
    [method, args]
  end
end
on_debug() { || ... } click to toggle source
# File lib/data_list_converter/base.rb, line 12
def on_debug
  self.debug = true
  yield
ensure
  self.debug = false
end
parameter(data, key, type) click to toggle source
# File lib/data_list_converter/helper.rb, line 76
def self.parameter(data, key, type)
  raise Exception, "`#{type}` should be hash, not `#{data.class}`: #{data}" unless data.kind_of?(Hash)
  raise Exception, "Need `#{key}` for `#{type}`, current: #{data}" if not data.has_key?(key)
  data.fetch(key)
end
recursive_flatten(out, data, header, sep, level, max_level) click to toggle source
# File lib/data_list_converter/helper.rb, line 65
def self.recursive_flatten(out, data, header, sep, level, max_level)
  data.each do |k, v|
    k = header ? :"#{header}#{sep}#{k}" : k
    if v.kind_of?(Hash) and (!max_level or level <= max_level)
      recursive_flatten(out, v, k, sep, level+1, max_level)
    else
      out[k] = v
    end
  end
end
register_converter(from_type, to_type, &block) click to toggle source

register_converter(:item_data, :item_iterator){ |data, options| … }

# File lib/data_list_converter/base.rb, line 25
def register_converter(from_type, to_type, &block)
  @route_map = nil # clear cache
  CONVERTERS[[from_type, to_type]] = block
end
register_filter(type, name, &block) click to toggle source
# File lib/data_list_converter/base.rb, line 30
def register_filter(type, name, &block)
  FILTERS[type] ||= {}
  FILTERS[type][name] = block
end
route_map() click to toggle source

convert adjacency list into quick lookup hash

# File lib/data_list_converter/base.rb, line 140
def route_map
  @route_map ||= \
  begin
    CONVERTERS.keys.
      inject({}) do |map, item|
      map[item.first] ||= []
      map[item.first] += [item[1]]
      map
    end
  end
end
routes() click to toggle source
# File lib/data_list_converter/helper.rb, line 27
def self.routes
  CONVERTERS.keys
end
save_to_file(filename, data, data_format=:item_data, options={}) click to toggle source
# File lib/data_list_converter/helper.rb, line 2
def self.save_to_file(filename, data, data_format=:item_data, options={})
  file_format = options.delete(:file_format) || self.get_file_format(filename)
  options[file_format] ||= {}
  options[file_format][:filename] = filename
  DataListConverter.convert(data_format, file_format, data, options)
end
types() click to toggle source
# File lib/data_list_converter/helper.rb, line 23
def self.types
  CONVERTERS.keys.flatten.uniq.sort
end
unify_item_data_keys(list) click to toggle source

sometimes item data keys don’t exactly same, like:

[{a: 12}, {b: 11}]

should update to:

[{a: 12, b: nil}, {a: nil, b: 11}]

so it can be convent to table data

# File lib/data_list_converter/helper.rb, line 43
def self.unify_item_data_keys(list)
  keys = Set.new
  list.each do |item|
    keys += item.keys
  end
  list.each do |item|
    keys.each do |key|
      item[key] ||= nil
    end
  end
end