class Textoken::Base

Inits options, findings and responds to tokens Does not raise error when text or options are nil Splits the text and makes it ready for other operations

Attributes

dont_split[R]
findings[R]
options[R]
text[R]

Public Class Methods

new(text, opt = nil) click to toggle source
# File lib/textoken/base.rb, line 8
def initialize(text, opt = nil)
  @text     = initial_split(text)
  @options  = Options.new(opt)
end

Public Instance Methods

tokens() click to toggle source

we do take intersection array of results returning from multiple options

# File lib/textoken/base.rb, line 15
def tokens
  options.collection.each do |option|
    if @findings.nil?
      @findings = option.tokenize(self)
    else
      @findings &= option.tokenize(self)
    end
  end

  Tokenizer.new(self).tokens
end
words() click to toggle source
# File lib/textoken/base.rb, line 27
def words
  # tokenize options but do not make the last split
  @dont_split = true
  tokens
end

Private Instance Methods

initial_split(text) click to toggle source
# File lib/textoken/base.rb, line 35
def initial_split(text)
  text ? text.split(' ') : []
end