class Lexer

Constants

TOKENS

Attributes

scanner[R]
tokens[R]

Public Class Methods

new(string) click to toggle source
# File bin/twitter-algebra, line 69
def initialize(string)
  @scanner = StringScanner.new(string)
  @tokens = []
end

Public Instance Methods

lex() click to toggle source
# File bin/twitter-algebra, line 74
def lex
  until scanner.eos?
    tokens << next_token
  end
  tokens.reject! { |token| token.type == :WHITESPACE }
  tokens << Token.new(:END, "")
end

Private Instance Methods

next_token() click to toggle source
# File bin/twitter-algebra, line 85
def next_token
  TOKENS.each do |type, regexp|
    if str = scanner.scan(regexp)
      return Token.new(type, str)
    end
  end
  raise SyntaxError, "Unexpected character: #{scanner.getch.inspect}"
end