class Rox::Core::StringTokenizer
Public Class Methods
new(text, delim, return_delims)
click to toggle source
# File lib/rox/core/roxx/string_tokenizer.rb, line 4 def initialize(text, delim, return_delims) @current_position = 0 @new_position = -1 @text = text @max_position = text.length @delimiters = delim @ret_delims = return_delims end
Public Instance Methods
delimiter?(char)
click to toggle source
# File lib/rox/core/roxx/string_tokenizer.rb, line 43 def delimiter?(char) @delimiters.include?(char) end
more_tokens?()
click to toggle source
# File lib/rox/core/roxx/string_tokenizer.rb, line 47 def more_tokens? @new_position = skip_delimiters(@current_position) @new_position < @max_position end
next_token(delim = nil)
click to toggle source
# File lib/rox/core/roxx/string_tokenizer.rb, line 52 def next_token(delim = nil) delims_changed = false unless delim.nil? @delimiters = delim delims_changed = true end @current_position = @new_position >= 0 && !delims_changed ? @new_position : skip_delimiters(@current_position) @new_position = -1 raise ArgumentError, 'invalid operation' if @current_position >= @max_position start = @current_position @current_position = scan_token(@current_position) @text[start...@current_position] end
scan_token(start_pos)
click to toggle source
# File lib/rox/core/roxx/string_tokenizer.rb, line 26 def scan_token(start_pos) position = start_pos while position < @max_position c = @text[position] break if delimiter?(c) position += 1 end if @ret_delims && start_pos == position c = @text[position] position += 1 if delimiter?(c) end position end
skip_delimiters(start_pos)
click to toggle source
# File lib/rox/core/roxx/string_tokenizer.rb, line 13 def skip_delimiters(start_pos) raise ArgumentError, 'delimiters is nil' if @delimiters.nil? position = start_pos while !@ret_delims && position < @max_position c = @text[position] break unless delimiter?(c) position += 1 end position end