class ShuntingYard::Lexer

Constants

SPACE_OR_EOL

Attributes

patterns[RW]
separator_pattern[RW]

Public Class Methods

new() click to toggle source
# File lib/shunting_yard/lexer.rb, line 10
def initialize
  @patterns = []
  @separator_pattern = SPACE_OR_EOL
end

Public Instance Methods

add_pattern(name, regex, evaluator = -> (lexeme) { lexeme } click to toggle source
# File lib/shunting_yard/lexer.rb, line 15
def add_pattern(name, regex, evaluator = -> (lexeme) { lexeme })
  @patterns << [name, regex, evaluator]
end
tokenize(input) click to toggle source
# File lib/shunting_yard/lexer.rb, line 19
def tokenize(input)
  sc = StringScanner.new(input)
  matches = []

  until sc.eos?
    last_match = nil

    @patterns.each do |name, regex, evaluator|
      match = sc.check(regex)
      next if match.nil?

      value = evaluator.(match)
      last_match = [name, match, value]
      break
    end

    if last_match.nil?
      unknown_token = sc.check_until(separator_pattern).sub(separator_pattern, "")
      raise UnknownTokenError.new(unknown_token, sc.pos + 1)
    end

    sc.pos += last_match[1].bytesize
    matches << build_token(last_match) unless last_match[2].nil?
  end

  matches
end

Private Instance Methods

build_token(args) click to toggle source
# File lib/shunting_yard/lexer.rb, line 49
def build_token(args)
  Token.new(*args)
end