class EyeOfNewt::Tokenizer

Constants

A
ANYTHING
NO_MATCH
NUMBER
OF
OR
TO_TASTE
WHITESPACE
WORD

Attributes

string[R]
unit_modifiers[R]
units[R]

Public Class Methods

new(string, units: EyeOfNewt.units.all, unit_modifiers: EyeOfNewt.units.unit_modifiers) click to toggle source
# File lib/eye_of_newt/tokenizer.rb, line 18
def initialize(string, units: EyeOfNewt.units.all, unit_modifiers: EyeOfNewt.units.unit_modifiers)
  @string = string
  @units = units
  @unit_modifiers = unit_modifiers
  @ss = StringScanner.new(string)
  @scan_text = nil
end

Public Instance Methods

next_token() click to toggle source
# File lib/eye_of_newt/tokenizer.rb, line 26
def next_token
  return if @ss.eos?

  @ss.scan(WHITESPACE)

  case
  when @scan_text && text = @ss.scan(/#{ANYTHING}\b/)
    @scan_text = false
    [:TEXT, text]
  when text = @ss.scan(NUMBER)
    [:NUMBER, text]
  when text = @ss.scan(/#{OF}\b/)
    [:OF, text]
  when text = @ss.scan(/#{A}\b/)
    [:A, text]
  when text = @ss.scan(/#{OR}\b/)
    [:OR, text]
  when text = @ss.scan(/#{TO_TASTE}\b/)
    [:TO_TASTE, text]
  when text = @ss.scan(/#{unit_matcher}\b/)
    [:UNIT, text]
  when text = @ss.scan(/#{unit_modifier}\b/)
    [:UNIT_MODIFIER, text]
  when text = @ss.scan(/#{WORD}\b/)
    [:WORD, text]
  else
    x = @ss.getch
    @scan_text = true if x == ',' || x == '('
    [x, x]
  end
end

Private Instance Methods

match_any(elements) click to toggle source
# File lib/eye_of_newt/tokenizer.rb, line 68
def match_any(elements)
  if elements.any?
    r = elements
      .sort
      .reverse
      .map{|u|Regexp.escape(u)}
      .join("|")
    Regexp.new(r)
  else
    NO_MATCH
  end
end
unit_matcher() click to toggle source
# File lib/eye_of_newt/tokenizer.rb, line 64
def unit_matcher
  @unit_matcher ||= match_any(units)
end
unit_modifier() click to toggle source
# File lib/eye_of_newt/tokenizer.rb, line 60
def unit_modifier
  @unit_modifier_matcher ||= match_any(unit_modifiers)
end