class Redata::Parser

Constants

COMMENT_REGEX
CURRENT_TIME_REGEX
ENDIF_REGEX
END_TIME_REGEX
IFNUL_REGEX
IF_REGEX
LOAD_PARAMS_REGEX
LOAD_REGEX
LOCALS_LIST_REGEX
LOCALS_REGEX
START_TIME_REGEX
TIME_OFFSET_REGEX

Public Class Methods

gen_adjust_query(config) click to toggle source
# File lib/redata/parser.rb, line 42
def self.gen_adjust_query(config)
        self.parse config.query_file, config.tmp_exec_file, ''
end
gen_checkout_query(config) click to toggle source
# File lib/redata/parser.rb, line 30
def self.gen_checkout_query(config)
        Log.error! "ERROR: Only could checkout data from view" unless config.type == :view

        File.open config.tmp_exec_file, 'w' do |f|
                f.puts "UNLOAD ('"
                f.puts "SELECT * FROM #{config.source_name}"
                f.puts "') to 's3://#{RED.s3['bucket']}/#{config.bucket_file}'"
                f.puts "CREDENTIALS 'aws_access_key_id=#{RED.s3['aws_access_key_id']};aws_secret_access_key=#{RED.s3['aws_secret_access_key']}'"
                f.puts "ESCAPE ALLOWOVERWRITE PARALLEL OFF DELIMITER AS '\\t';"
        end
end
gen_create_query(config) click to toggle source
# File lib/redata/parser.rb, line 16
def self.gen_create_query(config)
        if config.type == :table
                self.gen_table_query config
        elsif config.type == :view
                self.gen_view_query config
        end
end
gen_delete_query(config) click to toggle source
# File lib/redata/parser.rb, line 24
def self.gen_delete_query(config)
        File.open config.tmp_exec_file, 'w' do |f|
                f.puts "DROP #{config.type} #{config.source_name} #{RED.is_forced ? 'CASCADE' : 'RESTRICT'};"
        end
end

Private Class Methods

gen_table_query(config) click to toggle source
# File lib/redata/parser.rb, line 48
def self.gen_table_query(config)
        Log.error! "ERROR: Relation error" unless config.type == :table

        tmp_file = config.tmp_file_dir.join "#{config.source_name}.resql"
        temp_tables = self.parse config.query_file, tmp_file

        File.open config.tmp_exec_file, 'w' do |f|
                # print temp tables
                temp_tables.each do |name|
                        f.puts "CREATE TEMP TABLE #{name} AS ("
                        f.puts File.read(config.tmp_file_dir.join "#{name}.resql")
                        f.puts ");"
                end

                # print create or insert query
                if RED.is_append
                        f.puts "INSERT INTO #{config.source_name} ("
                elsif
                        f.puts "CREATE #{config.type} #{config.source_name} AS ("
                end
                f.puts File.read tmp_file
                f.puts ");"
        end
end
gen_view_query(config) click to toggle source
# File lib/redata/parser.rb, line 73
def self.gen_view_query(config)
        Log.error! "ERROR: Relation error" unless config.type == :view

        tmp_file = config.tmp_file_dir.join "#{config.source_name}.resql"
        temp_tables = self.parse config.query_file, tmp_file

        File.open config.tmp_exec_file, 'w' do |f|
                f.puts "CREATE #{config.type} #{config.source_name} AS ("
                temp_tables.each_with_index do |name, index|
                        f.puts "#{index == 0 ? 'WITH' : ','} #{name} AS ("
                        f.puts File.read(config.tmp_file_dir.join "#{name}.resql")
                        f.puts ")"
                end

                # print create query
                main = File.read tmp_file
                unless temp_tables.empty?
                        main.gsub! 'WITH', ','
                        main.gsub! 'with', ','
                end
                f.puts main
                f.puts ");"
        end
end
parse(in_file, out_file, skip_char=';') click to toggle source
# File lib/redata/parser.rb, line 98
def self.parse(in_file, out_file, skip_char=';')
        Log.error! "ERROR: Query file '#{in_file.relative_path_from RED.root}' not exists" unless in_file.exist?

        temp_tables = []
        parse_enable = true
        File.open out_file, 'w' do |out|
                File.open(in_file).each do |line|
                        # remove comments
                        line.gsub!(COMMENT_REGEX, '')
                        # remove skip_char
                        line.gsub!(skip_char, '')
                        # remove empty line
                        next if !line || line.empty? || line =~ /^\s*$/

                        # check if else condition
                        if line =~ IFNUL_REGEX
                                res = line.scan(IFNUL_REGEX).first
                                var = res[0]
                                parse_enable = RED.locals[var.to_sym].nil?
                                next
                        elsif line =~ IF_REGEX
                                res = line.scan(IF_REGEX).first
                                var = res[0]
                                val = res[1].gsub /[\s|\'|\"]+/, ''
                                parse_enable = (RED.locals[var.to_sym] == val)
                                next
                        elsif line =~ ENDIF_REGEX
                                parse_enable = true
                                next
                        end
                        next unless parse_enable

                        # compile sub file
                        if line =~ LOAD_REGEX
                                # parse load syntax
                                res = line.scan(LOAD_REGEX).first
                                sub = res[0].gsub /[\s|\'|\"]+/, ''
                                name = res[1].gsub /[\s|:]+/, ''
                                Log.error! "QUERY ERROR: syntax error for load query: #{line}" if sub.empty? || name.empty?
                                
                                local_params = {}
                                line.scan(LOAD_PARAMS_REGEX).each do |res|
                                        key = res[0].gsub(/\s/, '').to_sym
                                        val = res[1].gsub /\s/, ''
                                        local_params[key] = RED.locals[key]
                                        RED.locals[key] = val
                                end
                                sub_file = in_file.parent.join "_#{sub}.red.sql"
                                sub_file = RED.root.join 'query', 'shared', "_#{sub}.red.sql" unless sub_file.exist?
                                sub_temp_tables = self.parse sub_file, out_file.dirname.join("#{name}.resql")
                                sub_temp_tables.each do |n|
                                        temp_tables.push n unless temp_tables.include? n
                                end
                                temp_tables.push name unless temp_tables.include? name
                                local_params.each do |key, val|
                                        RED.locals[key] = val
                                end
                                local_params = {}
                                next  # load query line, can not contain other content
                        end

                        # parse [start_time] syntax
                        line.gsub! START_TIME_REGEX, "'#{RED.start_time}'" 
                        # parse [end_time] syntax
                        line.gsub! END_TIME_REGEX, "'#{RED.end_time}'"
                        # parse [current_time] syntax
                        line.gsub! CURRENT_TIME_REGEX, "'#{RED.current_time}'"

                        # parse [3 days ago]
                        res = line.scan(TIME_OFFSET_REGEX).each do |res|
                                line.gsub! "[#{res[0]} days ago]", "'#{RED.date_days_ago(res[0].to_i)}'"
                        end
                        # parse [locals] syntax
                        line.scan(LOCALS_REGEX).each do |res|
                                key = res.first
                                Log.error! "QUERY ERROR: Local params #{key} was missing." unless RED.locals[key.to_sym]
                                line.gsub! "[#{key}]", "'#{RED.locals[key.to_sym]}'"
                        end
                        # parse [<local_list>] syntax
                        line.scan(LOCALS_LIST_REGEX).each do |res|
                                key = res.first
                                Log.error! "QUERY ERROR: Local params #{key} was missing." unless RED.locals[key.to_sym]
                                line = line.gsub "[<#{key}>]", "(#{RED.locals[key.to_sym].split(',').map{|e| "'#{e}'"}.join(',')})"
                        end

                        out.puts line.gsub skip_char, ''
                end
        end
        temp_tables
end