module Lexer

Constants

Patterns

Public Class Methods

next_token(string, index) click to toggle source
# File lib/special-giggle/lexer.rb, line 35
def self.next_token(string, index)
  Patterns.each do |type, pattern|
    res = string.match(pattern, index)
    next if res.nil?
    return [res[0], type, res.end(0)] if res.begin(0) == index
  end
  [nil, nil, nil]
end
tokenize(string) click to toggle source
# File lib/special-giggle/lexer.rb, line 44
def self.tokenize(string)
  tokens = Array.new
  index, line, column = 0, 1, 1
  while index < string.length
    rstring, rtype, index = next_token(string, index)
    case rtype
    when nil
      raise LexerError, "Unrecognized character in line #{line}, column #{column}"
    when :Blank
      next
    when :NewLine
      line, column = line + 1, 1
    else
      tokens << Token.new(rstring, rtype)
    end
  end
  tokens
end