class Textoken::Base
Inits options, findings and responds to tokens Does not raise error when text or options are nil Splits the text and makes it ready for other operations
Attributes
dont_split[R]
findings[R]
options[R]
text[R]
Public Class Methods
new(text, opt = nil)
click to toggle source
# File lib/textoken/base.rb, line 8 def initialize(text, opt = nil) @text = initial_split(text) @options = Options.new(opt) end
Public Instance Methods
tokens()
click to toggle source
we do take intersection array of results returning from multiple options
# File lib/textoken/base.rb, line 15 def tokens options.collection.each do |option| if @findings.nil? @findings = option.tokenize(self) else @findings &= option.tokenize(self) end end Tokenizer.new(self).tokens end
words()
click to toggle source
# File lib/textoken/base.rb, line 27 def words # tokenize options but do not make the last split @dont_split = true tokens end
Private Instance Methods
initial_split(text)
click to toggle source
# File lib/textoken/base.rb, line 35 def initial_split(text) text ? text.split(' ') : [] end