class Tr8n::Tokenizers::Decoration
Constants
- RESERVED_TOKEN
- RE_LONG_TOKEN_END
- RE_LONG_TOKEN_START
- RE_SHORT_TOKEN_END
- RE_SHORT_TOKEN_START
- RE_TEXT
Attributes
context[R]
fragments[R]
opts[R]
text[R]
tokens[R]
Public Class Methods
new(text, context = {}, opts = {})
click to toggle source
# File lib/tr8n/tokenizers/decoration.rb, line 68 def initialize(text, context = {}, opts = {}) @text = "[#{RESERVED_TOKEN}]#{text}[/#{RESERVED_TOKEN}]" @context = context @opts = opts tokenize end
required?(label)
click to toggle source
# File lib/tr8n/tokenizers/decoration.rb, line 64 def self.required?(label) label.index('[') end
Public Instance Methods
allowed_token?(token)
click to toggle source
# File lib/tr8n/tokenizers/decoration.rb, line 146 def allowed_token?(token) return true if opts[:allowed_tokens].nil? opts[:allowed_tokens].include?(token) end
apply(token, value)
click to toggle source
# File lib/tr8n/tokenizers/decoration.rb, line 151 def apply(token, value) return value if token == RESERVED_TOKEN return value unless allowed_token?(token) method = context[token.to_sym] || context[token.to_s] if method if method.is_a?(Proc) return method.call(value) end if method.is_a?(Array) or method.is_a?(Hash) return default_decoration(token, value) end if method.is_a?(String) return method.to_s.gsub('{$0}', value) end Tr8n.logger.error("Invalid decoration token value for #{token} in #{text}") return value end if Tr8n.config.default_token_value(normalize_token(token), :decoration) return default_decoration(token, value) end Tr8n.logger.error("Missing decoration token value for #{token} in #{text}") value end
default_decoration(token_name, token_value)
click to toggle source
# File lib/tr8n/tokenizers/decoration.rb, line 124 def default_decoration(token_name, token_value) default_decoration = Tr8n.config.default_token_value(normalize_token(token_name), :decoration) unless default_decoration Tr8n.logger.error("Invalid decoration token value for #{token_name} in #{text}") return token_value end default_decoration = default_decoration.clone decoration_token_values = context[token_name.to_sym] || context[token_name.to_s] default_decoration.gsub!('{$0}', token_value.to_s) if decoration_token_values.is_a?(Hash) decoration_token_values.keys.each do |key| default_decoration.gsub!("{$#{key}}", decoration_token_values[key].to_s) end end default_decoration end
evaluate(expr)
click to toggle source
# File lib/tr8n/tokenizers/decoration.rb, line 186 def evaluate(expr) unless expr.is_a?(Array) return expr end token = expr[0] args = expr.drop(1) value = args.map { |a| self.evaluate(a) }.join('') apply(token, value) end
normalize_token(name)
click to toggle source
# File lib/tr8n/tokenizers/decoration.rb, line 182 def normalize_token(name) name.to_s.gsub(/(\d)*$/, '') end
parse()
click to toggle source
# File lib/tr8n/tokenizers/decoration.rb, line 85 def parse return @text unless fragments token = fragments.shift if token.match(/#{RE_SHORT_TOKEN_START}/) return parse_tree(token.gsub(/[\[:]/, ''), :short) end if token.match(/#{RE_LONG_TOKEN_START}/) return parse_tree(token.gsub(/[\[\]]/, ''), :long) end token.to_s end
parse_tree(name, type = :short)
click to toggle source
# File lib/tr8n/tokenizers/decoration.rb, line 100 def parse_tree(name, type = :short) tree = [name] @tokens << name unless (@tokens.include?(name) or name == RESERVED_TOKEN) if type == :short first = true until fragments.first.nil? or fragments.first.match(/#{RE_SHORT_TOKEN_END}/) value = parse if first and value.is_a?(String) value = value.lstrip first = false end tree << value end elsif type == :long until fragments.first.nil? or fragments.first.match(/#{RE_LONG_TOKEN_END}/) tree << parse end end fragments.shift tree end
substitute()
click to toggle source
# File lib/tr8n/tokenizers/decoration.rb, line 198 def substitute evaluate(parse) end
tokenize()
click to toggle source
# File lib/tr8n/tokenizers/decoration.rb, line 75 def tokenize re = [RE_SHORT_TOKEN_START, RE_SHORT_TOKEN_END, RE_LONG_TOKEN_START, RE_LONG_TOKEN_END, RE_TEXT].join('|') @fragments = text.scan(/#{re}/) @tokens = [] end