module Lexer
Constants
- Patterns
Public Class Methods
next_token(string, index)
click to toggle source
# File lib/special-giggle/lexer.rb, line 35 def self.next_token(string, index) Patterns.each do |type, pattern| res = string.match(pattern, index) next if res.nil? return [res[0], type, res.end(0)] if res.begin(0) == index end [nil, nil, nil] end
tokenize(string)
click to toggle source
# File lib/special-giggle/lexer.rb, line 44 def self.tokenize(string) tokens = Array.new index, line, column = 0, 1, 1 while index < string.length rstring, rtype, index = next_token(string, index) case rtype when nil raise LexerError, "Unrecognized character in line #{line}, column #{column}" when :Blank next when :NewLine line, column = line + 1, 1 else tokens << Token.new(rstring, rtype) end end tokens end