class DataListConverter
this class is used for convert data between types, so we can use it to easily export and import data.
item_data: [{name: ‘xx’, value: 12}, …] table_data: [[‘name’, ‘value’], [‘xx’, ‘12’], ..] item_iterator and table_iterator are iterators which yield each row
usage: require ‘data_list_converter/types/fast_xlsx_file’ d = [{a: 12.3, b: ‘xx’}] DataListConverter.save_to_file
(“t.xls”, d, :item_data, file_format: :fast_xlsx_file)
multi_sheet_table_data = {
'sheet1' => [columns, row, row, ...], 'sheet2' => [columns, row, row, ...],
} also: multi_sheet_item_data multi_sheet_table_iterator multi_sheet_item_iterator
ActiveRecords
Constants
- CONVERTERS
- FILTERS
- MULT_SHEET_CONVERTS
- VERSION
Attributes
Public Class Methods
Example: convert(:item_iterator, :item_data, iter) convert(:item_iterator, :csv_file, iter, csv_file: {filename: ‘result.csv’}) convert(:csv_file, :item_data, {filename: ‘result.csv’})
can add filter: filter = :limit filter = {limit: {size: 2}} filter = [{limit: {size: 12}}, {count: {size: 4}}] convert(:item_iterator, :table_data, iter, table_iterator: {filter: filter})
# File lib/data_list_converter/base.rb, line 45 def convert(from_type, to_type, from_value, options={}) methods = [] add_filter = lambda { |type| filters = (options[type] || {})[:filter] return unless filters methods += normalize_filters(type, filters) } route = find_route(from_type, to_type) add_filter.call(route[0]) self.log("route: #{route}") (0..(route.length-2)).map do |i| from_type, to_type = route[i], route[i+1] method = CONVERTERS[[from_type, to_type]] raise "cannot find converter #{from_type} -> #{to_type}" unless method self.log "#{from_type} -> #{to_type} options: #{options[to_type]}" methods.push([method, options[to_type] || {}]) add_filter.call(to_type) end self.log("methods: #{methods}") methods.inject(from_value) do |v, method| method, args = method method.call(v, args) end end
# File lib/data_list_converter/types/basic.rb, line 40 def self.data_to_iterator(data, options={}) lambda { |&block| data.each do |d| block.call(d) end } end
# File lib/data_list_converter/helper.rb, line 31 def self.file_types matcher = /(.*)_file$/ DataListConverter.types.select do |type| matcher.match(type) end end
One type of data can be converted into any other types, we have a list of convert methods: CONVERTERS
If we want to convert between types, like: convert item_data into csv_file, we need find all the intermidate data type, like: [:item_data, :item_iterator, :table_iterator, :csv_file]
# File lib/data_list_converter/base.rb, line 95 def find_route(from_type, to_type) [from_type, to_type].each do |type| raise Exception, "cannot find type: #{type}" unless self.types.include?(type) end raise Exception, "from_type should not equal to to_type: #{from_type}" if from_type == to_type # map wide search checked = Set.new checking = Set.new([from_type]) directions = {} while not checking.empty? current_node = checking.first next_nodes = route_map[current_node] # mark direction from from_type next_nodes.each do |node| # first marked is the shortest directions[node] ||= current_node end if next_nodes.include?(to_type) # get route start = to_type route = [start] while start != from_type previous = directions[start] raise "cannot find previous for #{start} in #{directions}" if not previous route.push(previous) start = previous end return route.reverse else checking.delete(current_node) checked.add(current_node) checking += Set.new(next_nodes) - checked end end log = ["Route not found: #{from_type} -> #{to_type}", "Current routes:"] log += self.routes.map{|from, to| "#{from} -> #{to}"} raise Exception, log.join("\n") end
flatten multi level item data into one level, example:
{a: {b: 12}, c: {d: {e: 11}}} => {:"a:b"=>12, :"c:d:e"=>11}
# File lib/data_list_converter/helper.rb, line 59 def self.flatten(data, sep=':', max_level=nil) out = {} recursive_flatten(out, data, nil, sep, 1, max_level) out end
# File lib/data_list_converter/helper.rb, line 15 def self.get_file_format(filename) file_type = (File.extname(filename)[1..-1] + "_file").to_sym unless DataListConverter.file_types.include?(file_type) raise "unknown file format: #{file_type}" end file_type end
# File lib/data_list_converter/filters/limit.rb, line 2 def self.iterator_limit(proc, options) limit_size = options[:size] || 10 lambda { |&block| limit = 0 proc.call do |item| block.call(item) limit += 1 break if limit >= limit_size end } end
# File lib/data_list_converter/types/basic.rb, line 32 def self.iterator_to_data(proc, options={}) out = [] proc.call { |d| out << d } out end
# File lib/data_list_converter/helper.rb, line 9 def self.load_from_file(filename, data_format=:item_data, options={}) file_format = options.delete(:file_format) || self.get_file_format(filename) options[:filename] = filename DataListConverter.convert(file_format, data_format, options) end
# File lib/data_list_converter/base.rb, line 19 def log(msg) return unless debug puts "#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}\t#{msg}" end
# File lib/data_list_converter/types/marshal.rb, line 10 def self.marshal_data_to_file(data, options) filename = self.parameter(options, :filename, :marshal) File.open(filename, 'w+') do |f| Marshal.dump(data, f) end options[:filename] end
# File lib/data_list_converter/types/marshal.rb, line 3 def self.marshal_file_to_data(input, options=nil) filename = self.parameter(input, :filename, :input) File.open(filename) do |f| Marshal.load(f) end end
# File lib/data_list_converter/base.rb, line 73 def normalize_filters(type, filters) # filter list as array filters = [filters] unless filters.kind_of?(Array) filters.map do |v| # fix filter arguments case v # {:limit, {count: 12}} => [:limit, {count: 12}] when Hash; v.first # :debug => [:debug, {}] when Symbol, String; [v, {}] else; v end end.map do |name, args| method = FILTERS[type][name] rescue raise("cannot find method for type #{type} filter #{name}") [method, args] end end
# File lib/data_list_converter/base.rb, line 12 def on_debug self.debug = true yield ensure self.debug = false end
# File lib/data_list_converter/helper.rb, line 76 def self.parameter(data, key, type) raise Exception, "`#{type}` should be hash, not `#{data.class}`: #{data}" unless data.kind_of?(Hash) raise Exception, "Need `#{key}` for `#{type}`, current: #{data}" if not data.has_key?(key) data.fetch(key) end
# File lib/data_list_converter/helper.rb, line 65 def self.recursive_flatten(out, data, header, sep, level, max_level) data.each do |k, v| k = header ? :"#{header}#{sep}#{k}" : k if v.kind_of?(Hash) and (!max_level or level <= max_level) recursive_flatten(out, v, k, sep, level+1, max_level) else out[k] = v end end end
register_converter
(:item_data, :item_iterator){ |data, options| … }
# File lib/data_list_converter/base.rb, line 25 def register_converter(from_type, to_type, &block) @route_map = nil # clear cache CONVERTERS[[from_type, to_type]] = block end
# File lib/data_list_converter/base.rb, line 30 def register_filter(type, name, &block) FILTERS[type] ||= {} FILTERS[type][name] = block end
convert adjacency list into quick lookup hash
# File lib/data_list_converter/base.rb, line 140 def route_map @route_map ||= \ begin CONVERTERS.keys. inject({}) do |map, item| map[item.first] ||= [] map[item.first] += [item[1]] map end end end
# File lib/data_list_converter/helper.rb, line 27 def self.routes CONVERTERS.keys end
# File lib/data_list_converter/helper.rb, line 2 def self.save_to_file(filename, data, data_format=:item_data, options={}) file_format = options.delete(:file_format) || self.get_file_format(filename) options[file_format] ||= {} options[file_format][:filename] = filename DataListConverter.convert(data_format, file_format, data, options) end
# File lib/data_list_converter/helper.rb, line 23 def self.types CONVERTERS.keys.flatten.uniq.sort end
sometimes item data keys don’t exactly same, like:
[{a: 12}, {b: 11}]
should update to:
[{a: 12, b: nil}, {a: nil, b: 11}]
so it can be convent to table data
# File lib/data_list_converter/helper.rb, line 43 def self.unify_item_data_keys(list) keys = Set.new list.each do |item| keys += item.keys end list.each do |item| keys.each do |key| item[key] ||= nil end end end