class TwitterCldr::Tokenizers::DateTokenizer
Attributes
data_reader[R]
Public Class Methods
new(data_reader)
click to toggle source
# File lib/twitter_cldr/tokenizers/calendars/date_tokenizer.rb, line 22 def initialize(data_reader) @data_reader = data_reader end
tokenizer()
click to toggle source
# File lib/twitter_cldr/tokenizers/calendars/date_tokenizer.rb, line 11 def tokenizer @tokenizer ||= Tokenizer.new([ TokenRecognizer.new(:composite, /^\#\{[^\}]+\}/, /^\#\{([^\}]+)\}/), TokenRecognizer.new(:pattern, /^(?:G{1,5}|y+|Y+|Q{1,4}|q{1,5}|M{1,5}|L{1,5}|d{1,2}|F{1}|E{1,5}|e{1,5}|c{1,5}|w{1,2}|W{1})/), TokenRecognizer.new(:plaintext, //) ], /(\s*\'[\w\s-]+\'\s*|G{1,5}|y+|Y+|Q{1,4}|q{1,5}|M{1,5}|L{1,5}|d{1,2}|F{1}|E{1,5}|e{1,5}|c{1,5}|w{1,2}|W{1}|\#\{[^\}]+\})/) end
Public Instance Methods
tokenize(pattern)
click to toggle source
# File lib/twitter_cldr/tokenizers/calendars/date_tokenizer.rb, line 26 def tokenize(pattern) PatternTokenizer.new(data_reader, tokenizer).tokenize(pattern) end
tokenizer()
click to toggle source
# File lib/twitter_cldr/tokenizers/calendars/date_tokenizer.rb, line 30 def tokenizer self.class.tokenizer end