class NScript::Rewriter
Constants
- BALANCED_PAIRS
- EXPRESSION_CLOSE
- EXPRESSION_START
- EXPRESSION_TAIL
- IMPLICIT_CALL
- IMPLICIT_END
- IMPLICIT_FUNC
- INVERSES
- SINGLE_CLOSERS
- SINGLE_LINERS
Public Instance Methods
add_implicit_indentation()
click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 145 def add_implicit_indentation scan_tokens do |prev, token, post, i| next 1 unless SINGLE_LINERS.include?(token[0]) && post[0] != :INDENT && !(token[0] == :ELSE && post[0] == :IF) # Elsifs shouldn't get blocks. starter = token[0] line = token[1].line @tokens.insert(i + 1, [:INDENT, Value.new(2, line)]) idx = i + 1 parens = 0 loop do idx += 1 tok = @tokens[idx] if (!tok || SINGLE_CLOSERS.include?(tok[0]) || (tok[0] == ')' && parens == 0)) && !(starter == :ELSE && tok[0] == :ELSE) insertion = @tokens[idx - 1][0] == "," ? idx - 1 : idx @tokens.insert(insertion, [:OUTDENT, Value.new(2, line)]) break end parens += 1 if tok[0] == '(' parens -= 1 if tok[0] == ')' end next 1 unless token[0] == :THEN @tokens.delete_at(i) next 0 end end
add_implicit_parentheses()
click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 124 def add_implicit_parentheses stack = [0] scan_tokens do |prev, token, post, i| stack.push(0) if token[0] == :INDENT if token[0] == :OUTDENT last = stack.pop stack[-1] += last end if stack.last > 0 && (IMPLICIT_END.include?(token[0]) || post.nil?) idx = token[0] == :OUTDENT ? i + 1 : i stack.last.times { @tokens.insert(idx, [:CALL_END, Value.new(')', token[1].line)]) } size, stack[-1] = stack[-1] + 1, 0 next size end next 1 unless IMPLICIT_FUNC.include?(prev[0]) && IMPLICIT_CALL.include?(token[0]) @tokens.insert(i, [:CALL_START, Value.new('(', token[1].line)]) stack[-1] += 1 next 2 end end
adjust_comments()
click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 51 def adjust_comments scan_tokens do |prev, token, post, i| next 1 unless token[0] == :COMMENT before, after = @tokens[i - 2], @tokens[i + 2] if before && after && ((before[0] == :INDENT && after[0] == :OUTDENT) || (before[0] == :OUTDENT && after[0] == :INDENT)) && before[1] == after[1] @tokens.delete_at(i + 2) @tokens.delete_at(i - 2) next 0 elsif prev[0] == "\n" && [:INDENT].include?(after[0]) @tokens.delete_at(i + 2) @tokens[i - 1] = after next 1 elsif !["\n", :INDENT, :OUTDENT].include?(prev[0]) @tokens.insert(i, ["\n", Value.new("\n", token[1].line)]) next 2 else next 1 end end end
close_open_calls_and_indexes()
click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 97 def close_open_calls_and_indexes parens, brackets = [0], [0] scan_tokens do |prev, token, post, i| case token[0] when :CALL_START then parens.push(0) when :INDEX_START then brackets.push(0) when '(' then parens[-1] += 1 when '[' then brackets[-1] += 1 when ')' if parens.last == 0 parens.pop token[0] = :CALL_END else parens[-1] -= 1 end when ']' if brackets.last == 0 brackets.pop token[0] = :INDEX_END else brackets[-1] -= 1 end end next 1 end end
ensure_balance(*pairs)
click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 173 def ensure_balance(*pairs) puts "\nbefore ensure_balance: #{@tokens.inspect}" if ENV['VERBOSE'] levels, lines = Hash.new(0), Hash.new scan_tokens do |prev, token, post, i| pairs.each do |pair| open, close = *pair levels[open] += 1 if token[0] == open levels[open] -= 1 if token[0] == close lines[token[0]] = token[1].line raise ParseError.new(token[0], token[1], nil) if levels[open] < 0 end next 1 end unclosed = levels.detect {|k, v| v > 0 } sym = unclosed && unclosed[0] raise ParseError.new(sym, Value.new(sym, lines[sym]), nil, "unclosed '#{sym}'") if unclosed end
move_commas_outside_outdents()
click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 87 def move_commas_outside_outdents scan_tokens do |prev, token, post, i| if token[0] == :OUTDENT && prev[0] == ',' @tokens.delete_at(i) @tokens.insert(i - 1, token) end next 1 end end
remove_leading_newlines()
click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 75 def remove_leading_newlines @tokens.shift if @tokens[0][0] == "\n" end
remove_mid_expression_newlines()
click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 79 def remove_mid_expression_newlines scan_tokens do |prev, token, post, i| next 1 unless post && EXPRESSION_CLOSE.include?(post[0]) && token[0] == "\n" @tokens.delete_at(i) next 0 end end
rewrite(tokens)
click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 28 def rewrite(tokens) @tokens = tokens adjust_comments remove_leading_newlines remove_mid_expression_newlines move_commas_outside_outdents close_open_calls_and_indexes add_implicit_parentheses add_implicit_indentation ensure_balance(*BALANCED_PAIRS) rewrite_closing_parens @tokens end
rewrite_closing_parens()
click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 191 def rewrite_closing_parens verbose = ENV['VERBOSE'] stack, debt = [], Hash.new(0) stack_stats = lambda { "stack: #{stack.inspect} debt: #{debt.inspect}\n\n" } puts "rewrite_closing_original: #{@tokens.inspect}" if verbose scan_tokens do |prev, token, post, i| tag, inv = token[0], INVERSES[token[0]] # Push openers onto the stack. if EXPRESSION_START.include?(tag) stack.push(token) puts "pushing #{tag} #{stack_stats[]}" if verbose next 1 # The end of an expression, check stack and debt for a pair. elsif EXPRESSION_TAIL.include?(tag) puts @tokens[i..-1].inspect if verbose # If the tag is already in our debt, swallow it. if debt[inv] > 0 debt[inv] -= 1 @tokens.delete_at(i) puts "tag in debt #{tag} #{stack_stats[]}" if verbose next 0 else # Pop the stack of open delimiters. match = stack.pop mtag = match[0] # Continue onwards if it's the expected tag. if tag == INVERSES[mtag] puts "expected tag #{tag} #{stack_stats[]}" if verbose next 1 else # Unexpected close, insert correct close, adding to the debt. debt[mtag] += 1 puts "unexpected #{tag}, replacing with #{INVERSES[mtag]} #{stack_stats[]}" if verbose val = mtag == :INDENT ? match[1] : INVERSES[mtag] @tokens.insert(i, [INVERSES[mtag], Value.new(val, token[1].line)]) next 1 end end else # Uninteresting token: next 1 end end end
scan_tokens() { |tokens, tokens, tokens, i| ... }
click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 42 def scan_tokens i = 0 loop do break unless @tokens[i] move = yield(@tokens[i - 1], @tokens[i], @tokens[i + 1], i) i += move end end