class EyeOfNewt::Tokenizer
Constants
- A
- ANYTHING
- NO_MATCH
- NUMBER
- OF
- OR
- TO_TASTE
- WHITESPACE
- WORD
Attributes
string[R]
unit_modifiers[R]
units[R]
Public Class Methods
new(string, units: EyeOfNewt.units.all, unit_modifiers: EyeOfNewt.units.unit_modifiers)
click to toggle source
# File lib/eye_of_newt/tokenizer.rb, line 18 def initialize(string, units: EyeOfNewt.units.all, unit_modifiers: EyeOfNewt.units.unit_modifiers) @string = string @units = units @unit_modifiers = unit_modifiers @ss = StringScanner.new(string) @scan_text = nil end
Public Instance Methods
next_token()
click to toggle source
# File lib/eye_of_newt/tokenizer.rb, line 26 def next_token return if @ss.eos? @ss.scan(WHITESPACE) case when @scan_text && text = @ss.scan(/#{ANYTHING}\b/) @scan_text = false [:TEXT, text] when text = @ss.scan(NUMBER) [:NUMBER, text] when text = @ss.scan(/#{OF}\b/) [:OF, text] when text = @ss.scan(/#{A}\b/) [:A, text] when text = @ss.scan(/#{OR}\b/) [:OR, text] when text = @ss.scan(/#{TO_TASTE}\b/) [:TO_TASTE, text] when text = @ss.scan(/#{unit_matcher}\b/) [:UNIT, text] when text = @ss.scan(/#{unit_modifier}\b/) [:UNIT_MODIFIER, text] when text = @ss.scan(/#{WORD}\b/) [:WORD, text] else x = @ss.getch @scan_text = true if x == ',' || x == '(' [x, x] end end
Private Instance Methods
match_any(elements)
click to toggle source
# File lib/eye_of_newt/tokenizer.rb, line 68 def match_any(elements) if elements.any? r = elements .sort .reverse .map{|u|Regexp.escape(u)} .join("|") Regexp.new(r) else NO_MATCH end end
unit_matcher()
click to toggle source
# File lib/eye_of_newt/tokenizer.rb, line 64 def unit_matcher @unit_matcher ||= match_any(units) end
unit_modifier()
click to toggle source
# File lib/eye_of_newt/tokenizer.rb, line 60 def unit_modifier @unit_modifier_matcher ||= match_any(unit_modifiers) end