class TwitterCldr::Tokenizers::DateTimeTokenizer
Attributes
data_reader[R]
Public Class Methods
new(data_reader)
click to toggle source
# File lib/twitter_cldr/tokenizers/calendars/date_time_tokenizer.rb, line 23 def initialize(data_reader) @data_reader = data_reader end
tokenizer()
click to toggle source
# File lib/twitter_cldr/tokenizers/calendars/date_time_tokenizer.rb, line 11 def tokenizer @tokenizer ||= Tokenizer.new([ TokenRecognizer.new(:date, /\{\{date\}\}/), TokenRecognizer.new(:time, /\{\{time\}\}/), TokenRecognizer.new(:plaintext, /'.*'/), TokenRecognizer.new(:plaintext, //) ]) end
Public Instance Methods
full_tokenize(pattern)
click to toggle source
Tokenizes mixed date and time pattern strings, used to tokenize the additional date format patterns.
# File lib/twitter_cldr/tokenizers/calendars/date_time_tokenizer.rb, line 35 def full_tokenize(pattern) PatternTokenizer.new(data_reader, full_tokenizer).tokenize(pattern) end
tokenize(pattern)
click to toggle source
# File lib/twitter_cldr/tokenizers/calendars/date_time_tokenizer.rb, line 27 def tokenize(pattern) expand_tokens( PatternTokenizer.new(data_reader, tokenizer).tokenize(pattern) ) end
Protected Instance Methods
expand_date(token)
click to toggle source
# File lib/twitter_cldr/tokenizers/calendars/date_time_tokenizer.rb, line 54 def expand_date(token) date_reader = data_reader.date_reader date_reader.tokenizer.tokenize(date_reader.pattern) end
expand_time(token)
click to toggle source
# File lib/twitter_cldr/tokenizers/calendars/date_time_tokenizer.rb, line 59 def expand_time(token) time_reader = data_reader.time_reader time_reader.tokenizer.tokenize(time_reader.pattern) end
expand_tokens(tokens)
click to toggle source
# File lib/twitter_cldr/tokenizers/calendars/date_time_tokenizer.rb, line 41 def expand_tokens(tokens) tokens.inject([]) do |ret, token| ret + case token.type when :date expand_date(token) when :time expand_time(token) else [token] end end end
full_tokenizer()
click to toggle source
# File lib/twitter_cldr/tokenizers/calendars/date_time_tokenizer.rb, line 64 def full_tokenizer @@full_tokenizer ||= begin new_tok = Tokenizer.union( data_reader.date_reader.tokenizer.tokenizer, data_reader.time_reader.tokenizer.tokenizer ) do |recognizer| recognizer.token_type != :plaintext end new_tok.recognizers << TokenRecognizer.new(:plaintext, //) new_tok end end
tokenizer()
click to toggle source
# File lib/twitter_cldr/tokenizers/calendars/date_time_tokenizer.rb, line 78 def tokenizer self.class.tokenizer end