class Tr8n::Tokenizers::Data

Attributes

context[RW]
opts[RW]
text[RW]
tokens[RW]

Public Class Methods

new(text, context={}, opts={}) click to toggle source
# File lib/tr8n/tokenizers/data.rb, line 64
def initialize(text, context={}, opts={})
  self.text = text
  self.context = context
  self.opts = opts
  self.tokens = []
  tokenize
end
required?(label) click to toggle source
# File lib/tr8n/tokenizers/data.rb, line 60
def self.required?(label)
  label.index("{")
end
supported_tokens() click to toggle source
# File lib/tr8n/tokenizers/data.rb, line 56
def self.supported_tokens
  [Tr8n::Tokens::Data, Tr8n::Tokens::Method, Tr8n::Tokens::Transform]
end

Public Instance Methods

substitute(language, options = {}) click to toggle source
# File lib/tr8n/tokenizers/data.rb, line 85
def substitute(language, options = {})
  label = self.text
  tokens.each do |token|
    next unless token_allowed?(token)
    label = token.substitute(label, context, language, options)
  end
  label
end
token_allowed?(token) click to toggle source
# File lib/tr8n/tokenizers/data.rb, line 80
def token_allowed?(token)
  return true unless opts[:allowed_tokens]
  not opts[:allowed_tokens][token.name].nil?
end
tokenize() click to toggle source
# File lib/tr8n/tokenizers/data.rb, line 72
def tokenize
  self.tokens = []
  self.class.supported_tokens.each do |klass|
    self.tokens << klass.parse(self.text)
  end
  self.tokens.flatten!.uniq!
end