class NScript::Rewriter

Constants

BALANCED_PAIRS
EXPRESSION_CLOSE
EXPRESSION_START
EXPRESSION_TAIL
IMPLICIT_CALL
IMPLICIT_END
IMPLICIT_FUNC
INVERSES
SINGLE_CLOSERS
SINGLE_LINERS

Public Instance Methods

add_implicit_indentation() click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 145
def add_implicit_indentation
  scan_tokens do |prev, token, post, i|
    next 1 unless SINGLE_LINERS.include?(token[0]) && post[0] != :INDENT &&
      !(token[0] == :ELSE && post[0] == :IF) # Elsifs shouldn't get blocks.
    starter = token[0]
    line = token[1].line
    @tokens.insert(i + 1, [:INDENT, Value.new(2, line)])
    idx = i + 1
    parens = 0
    loop do
      idx += 1
      tok = @tokens[idx]
      if (!tok || SINGLE_CLOSERS.include?(tok[0]) ||
          (tok[0] == ')' && parens == 0)) &&
          !(starter == :ELSE && tok[0] == :ELSE)
        insertion = @tokens[idx - 1][0] == "," ? idx - 1 : idx
        @tokens.insert(insertion, [:OUTDENT, Value.new(2, line)])
        break
      end
      parens += 1 if tok[0] == '('
      parens -= 1 if tok[0] == ')'
    end
    next 1 unless token[0] == :THEN
    @tokens.delete_at(i)
    next 0
  end
end
add_implicit_parentheses() click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 124
def add_implicit_parentheses
  stack = [0]
  scan_tokens do |prev, token, post, i|
    stack.push(0) if token[0] == :INDENT
    if token[0] == :OUTDENT
      last = stack.pop
      stack[-1] += last
    end
    if stack.last > 0 && (IMPLICIT_END.include?(token[0]) || post.nil?)
      idx = token[0] == :OUTDENT ? i + 1 : i
      stack.last.times { @tokens.insert(idx, [:CALL_END, Value.new(')', token[1].line)]) }
      size, stack[-1] = stack[-1] + 1, 0
      next size
    end
    next 1 unless IMPLICIT_FUNC.include?(prev[0]) && IMPLICIT_CALL.include?(token[0])
    @tokens.insert(i, [:CALL_START, Value.new('(', token[1].line)])
    stack[-1] += 1
    next 2
  end
end
adjust_comments() click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 51
def adjust_comments
  scan_tokens do |prev, token, post, i|
    next 1 unless token[0] == :COMMENT
    before, after = @tokens[i - 2], @tokens[i + 2]
    if before && after &&
        ((before[0] == :INDENT && after[0] == :OUTDENT) ||
        (before[0] == :OUTDENT && after[0] == :INDENT)) &&
        before[1] == after[1]
      @tokens.delete_at(i + 2)
      @tokens.delete_at(i - 2)
      next 0
    elsif prev[0] == "\n" && [:INDENT].include?(after[0])
      @tokens.delete_at(i + 2)
      @tokens[i - 1] = after
      next 1
    elsif !["\n", :INDENT, :OUTDENT].include?(prev[0])
      @tokens.insert(i, ["\n", Value.new("\n", token[1].line)])
      next 2
    else
      next 1
    end
  end
end
close_open_calls_and_indexes() click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 97
def close_open_calls_and_indexes
  parens, brackets = [0], [0]
  scan_tokens do |prev, token, post, i|
    case token[0]
    when :CALL_START  then parens.push(0)
    when :INDEX_START then brackets.push(0)
    when '('          then parens[-1] += 1
    when '['          then brackets[-1] += 1
    when ')'
      if parens.last == 0
        parens.pop
        token[0] = :CALL_END
      else
        parens[-1] -= 1
      end
    when ']'
      if brackets.last == 0
        brackets.pop
        token[0] = :INDEX_END
      else
        brackets[-1] -= 1
      end
    end
    next 1
  end
end
ensure_balance(*pairs) click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 173
def ensure_balance(*pairs)
  puts "\nbefore ensure_balance: #{@tokens.inspect}" if ENV['VERBOSE']
  levels, lines = Hash.new(0), Hash.new
  scan_tokens do |prev, token, post, i|
    pairs.each do |pair|
      open, close = *pair
      levels[open] += 1 if token[0] == open
      levels[open] -= 1 if token[0] == close
      lines[token[0]] = token[1].line
      raise ParseError.new(token[0], token[1], nil) if levels[open] < 0
    end
    next 1
  end
  unclosed = levels.detect {|k, v| v > 0 }
  sym = unclosed && unclosed[0]
  raise ParseError.new(sym, Value.new(sym, lines[sym]), nil, "unclosed '#{sym}'") if unclosed
end
move_commas_outside_outdents() click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 87
def move_commas_outside_outdents
  scan_tokens do |prev, token, post, i|
    if token[0] == :OUTDENT && prev[0] == ','
      @tokens.delete_at(i)
      @tokens.insert(i - 1, token)
    end
    next 1
  end
end
remove_leading_newlines() click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 75
def remove_leading_newlines
  @tokens.shift if @tokens[0][0] == "\n"
end
remove_mid_expression_newlines() click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 79
def remove_mid_expression_newlines
  scan_tokens do |prev, token, post, i|
    next 1 unless post && EXPRESSION_CLOSE.include?(post[0]) && token[0] == "\n"
    @tokens.delete_at(i)
    next 0
  end
end
rewrite(tokens) click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 28
def rewrite(tokens)
  @tokens = tokens
  adjust_comments
  remove_leading_newlines
  remove_mid_expression_newlines
  move_commas_outside_outdents
  close_open_calls_and_indexes
  add_implicit_parentheses
  add_implicit_indentation
  ensure_balance(*BALANCED_PAIRS)
  rewrite_closing_parens
  @tokens
end
rewrite_closing_parens() click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 191
def rewrite_closing_parens
  verbose = ENV['VERBOSE']
  stack, debt = [], Hash.new(0)
  stack_stats = lambda { "stack: #{stack.inspect} debt: #{debt.inspect}\n\n" }
  puts "rewrite_closing_original: #{@tokens.inspect}" if verbose
  scan_tokens do |prev, token, post, i|
    tag, inv = token[0], INVERSES[token[0]]
    # Push openers onto the stack.
    if EXPRESSION_START.include?(tag)
      stack.push(token)
      puts "pushing #{tag} #{stack_stats[]}" if verbose
      next 1
    # The end of an expression, check stack and debt for a pair.
    elsif EXPRESSION_TAIL.include?(tag)
      puts @tokens[i..-1].inspect if verbose
      # If the tag is already in our debt, swallow it.
      if debt[inv] > 0
        debt[inv] -= 1
        @tokens.delete_at(i)
        puts "tag in debt #{tag} #{stack_stats[]}" if verbose
        next 0
      else
        # Pop the stack of open delimiters.
        match = stack.pop
        mtag  = match[0]
        # Continue onwards if it's the expected tag.
        if tag == INVERSES[mtag]
          puts "expected tag #{tag} #{stack_stats[]}" if verbose
          next 1
        else
          # Unexpected close, insert correct close, adding to the debt.
          debt[mtag] += 1
          puts "unexpected #{tag}, replacing with #{INVERSES[mtag]} #{stack_stats[]}" if verbose
          val = mtag == :INDENT ? match[1] : INVERSES[mtag]
          @tokens.insert(i, [INVERSES[mtag], Value.new(val, token[1].line)])
          next 1
        end
      end
    else
      # Uninteresting token:
      next 1
    end
  end
end
scan_tokens() { |tokens, tokens, tokens, i| ... } click to toggle source
# File lib/nscript/lexer/rewriter.rb, line 42
def scan_tokens
  i = 0
  loop do
    break unless @tokens[i]
    move = yield(@tokens[i - 1], @tokens[i], @tokens[i + 1], i)
    i += move
  end
end