class Rox::Core::StringTokenizer

Public Class Methods

new(text, delim, return_delims) click to toggle source
# File lib/rox/core/roxx/string_tokenizer.rb, line 4
def initialize(text, delim, return_delims)
  @current_position = 0
  @new_position = -1
  @text = text
  @max_position = text.length
  @delimiters = delim
  @ret_delims = return_delims
end

Public Instance Methods

delimiter?(char) click to toggle source
# File lib/rox/core/roxx/string_tokenizer.rb, line 43
def delimiter?(char)
  @delimiters.include?(char)
end
more_tokens?() click to toggle source
# File lib/rox/core/roxx/string_tokenizer.rb, line 47
def more_tokens?
  @new_position = skip_delimiters(@current_position)
  @new_position < @max_position
end
next_token(delim = nil) click to toggle source
# File lib/rox/core/roxx/string_tokenizer.rb, line 52
def next_token(delim = nil)
  delims_changed = false
  unless delim.nil?
    @delimiters = delim
    delims_changed = true
  end

  @current_position = @new_position >= 0 && !delims_changed ? @new_position : skip_delimiters(@current_position)
  @new_position = -1

  raise ArgumentError, 'invalid operation' if @current_position >= @max_position

  start = @current_position
  @current_position = scan_token(@current_position)
  @text[start...@current_position]
end
scan_token(start_pos) click to toggle source
# File lib/rox/core/roxx/string_tokenizer.rb, line 26
def scan_token(start_pos)
  position = start_pos
  while position < @max_position
    c = @text[position]
    break if delimiter?(c)

    position += 1
  end

  if @ret_delims && start_pos == position
    c = @text[position]
    position += 1 if delimiter?(c)
  end

  position
end
skip_delimiters(start_pos) click to toggle source
# File lib/rox/core/roxx/string_tokenizer.rb, line 13
def skip_delimiters(start_pos)
  raise ArgumentError, 'delimiters is nil' if @delimiters.nil?

  position = start_pos
  while !@ret_delims && position < @max_position
    c = @text[position]
    break unless delimiter?(c)

    position += 1
  end
  position
end