class TfIdfSimilarity::Tokenizer

Public Instance Methods

tokenize(text) click to toggle source

Tokenizes a text.

@param [String] text @return [Enumerator] an enumerator of Token objects

# File lib/tf-idf-similarity/tokenizer.rb, line 13
def tokenize(text)
  UnicodeUtils.each_word(text).map do |word|
    Token.new(word)
  end
end