module Splitta::WordTokenizer

Constants

TOKENIZE_REGEXPS

Public Instance Methods

tokenize(text) click to toggle source

Tokenize a string using the rules above

# File lib/splitta/word_tokenizer.rb, line 62
def tokenize(text)
  text = text.dup
  TOKENIZE_REGEXPS.each_slice(2) do |regexp, repl|
    text.gsub!(regexp, repl)
  end
  text
end