module ModelTokenizer::Base
Constants
- CHARSET
Public Instance Methods
has_token(*attributes)
click to toggle source
# File lib/model_tokenizer/base.rb, line 18 def has_token(*attributes) options = { :length => @@model_tokenizer_token_length }.merge!(attributes.last.is_a?(Hash) ? attributes.pop : {}) if(!options[:length].is_a?(Integer) || options[:length] < 8) options[:length] = @@model_tokenizer_token_length end @@model_tokenizer_token_length = options[:length] include InstanceMethods end
model_tokenizer_token_length()
click to toggle source
# File lib/model_tokenizer/base.rb, line 14 def model_tokenizer_token_length @@model_tokenizer_token_length end