module Splitta::WordTokenizer
Constants
- TOKENIZE_REGEXPS
Public Instance Methods
tokenize(text)
click to toggle source
Tokenize a string using the rules above
# File lib/splitta/word_tokenizer.rb, line 62 def tokenize(text) text = text.dup TOKENIZE_REGEXPS.each_slice(2) do |regexp, repl| text.gsub!(regexp, repl) end text end