class Tml::Tokenizers::Data
Attributes
context[RW]
opts[RW]
text[RW]
tokens[RW]
Public Class Methods
new(text, context={}, opts={})
click to toggle source
# File lib/tml/tokenizers/data.rb, line 64 def initialize(text, context={}, opts={}) self.text = text self.context = context self.opts = opts self.tokens = [] tokenize end
required?(label)
click to toggle source
# File lib/tml/tokenizers/data.rb, line 60 def self.required?(label) label.index('{') end
supported_tokens()
click to toggle source
# File lib/tml/tokenizers/data.rb, line 56 def self.supported_tokens [Tml::Tokens::Data, Tml::Tokens::Method, Tml::Tokens::Transform, Tml::Tokens::Map] end
Public Instance Methods
substitute(language, options = {})
click to toggle source
# File lib/tml/tokenizers/data.rb, line 85 def substitute(language, options = {}) label = self.text tokens.each do |token| next unless token_allowed?(token) label = token.substitute(label, context, language, options) end label end
token_allowed?(token)
click to toggle source
# File lib/tml/tokenizers/data.rb, line 80 def token_allowed?(token) return true unless opts[:allowed_tokens] not opts[:allowed_tokens][token.name].nil? end
tokenize()
click to toggle source
# File lib/tml/tokenizers/data.rb, line 72 def tokenize self.tokens = [] self.class.supported_tokens.each do |klass| self.tokens << klass.parse(self.text) end self.tokens.flatten!.uniq! end