class TwitterCldr::Transforms::Tokenizer

Public Instance Methods

tokenize(rule_text) click to toggle source
# File lib/twitter_cldr/transforms/tokenizer.rb, line 10
def tokenize(rule_text)
  tokenizer.tokenize(rule_text)
end

Private Instance Methods

recognizers() click to toggle source
# File lib/twitter_cldr/transforms/tokenizer.rb, line 20
def recognizers
  @recognizers ||= [
    TwitterCldr::Tokenizers::TokenRecognizer.new(:capture, /\$[\d]+/),
    TwitterCldr::Tokenizers::TokenRecognizer.new(:variable, /\$[\w]+/),
    TwitterCldr::Tokenizers::TokenRecognizer.new(:doubled_quote, /''/),
    TwitterCldr::Tokenizers::TokenRecognizer.new(:quoted_string, /'[^']*'/),
    TwitterCldr::Tokenizers::TokenRecognizer.new(:direction, /[<>]{1,2}/),
    TwitterCldr::Tokenizers::TokenRecognizer.new(:before_context, /[{]/),
    TwitterCldr::Tokenizers::TokenRecognizer.new(:after_context, /[}]/),
    TwitterCldr::Tokenizers::TokenRecognizer.new(:cursor, /\|/),
    TwitterCldr::Tokenizers::TokenRecognizer.new(:unicode_char, /\\u[a-fA-F0-9]{1,6}/),
    TwitterCldr::Tokenizers::TokenRecognizer.new(:unicode_char, /\\u\{[a-fA-F0-9]{1,6}\}/),
    TwitterCldr::Tokenizers::TokenRecognizer.new(:escaped_char, /\\./),
    TwitterCldr::Tokenizers::TokenRecognizer.new(:escaped_backslash, /\\\\/),
    TwitterCldr::Tokenizers::TokenRecognizer.new(:equals, /=/),
    TwitterCldr::Tokenizers::TokenRecognizer.new(:semicolon, /;/),
    TwitterCldr::Tokenizers::TokenRecognizer.new(:string, /[^ ]{1}/)
  ]
end
tokenizer() click to toggle source
# File lib/twitter_cldr/transforms/tokenizer.rb, line 16
def tokenizer
  TwitterCldr::Tokenizers::Tokenizer.new(recognizers)
end