class ShuntingYard::Lexer
Constants
- SPACE_OR_EOL
Attributes
patterns[RW]
separator_pattern[RW]
Public Class Methods
new()
click to toggle source
# File lib/shunting_yard/lexer.rb, line 10 def initialize @patterns = [] @separator_pattern = SPACE_OR_EOL end
Public Instance Methods
add_pattern(name, regex, evaluator = -> (lexeme) { lexeme }
click to toggle source
# File lib/shunting_yard/lexer.rb, line 15 def add_pattern(name, regex, evaluator = -> (lexeme) { lexeme }) @patterns << [name, regex, evaluator] end
tokenize(input)
click to toggle source
# File lib/shunting_yard/lexer.rb, line 19 def tokenize(input) sc = StringScanner.new(input) matches = [] until sc.eos? last_match = nil @patterns.each do |name, regex, evaluator| match = sc.check(regex) next if match.nil? value = evaluator.(match) last_match = [name, match, value] break end if last_match.nil? unknown_token = sc.check_until(separator_pattern).sub(separator_pattern, "") raise UnknownTokenError.new(unknown_token, sc.pos + 1) end sc.pos += last_match[1].bytesize matches << build_token(last_match) unless last_match[2].nil? end matches end
Private Instance Methods
build_token(args)
click to toggle source
# File lib/shunting_yard/lexer.rb, line 49 def build_token(args) Token.new(*args) end