class TwitterCldr::Tokenizers::DateTimeTokenizer

Attributes

data_reader[R]

Public Class Methods

new(data_reader) click to toggle source
# File lib/twitter_cldr/tokenizers/calendars/date_time_tokenizer.rb, line 23
def initialize(data_reader)
  @data_reader = data_reader
end
tokenizer() click to toggle source
# File lib/twitter_cldr/tokenizers/calendars/date_time_tokenizer.rb, line 11
def tokenizer
  @tokenizer ||= Tokenizer.new([
    TokenRecognizer.new(:date, /\{\{date\}\}/),
    TokenRecognizer.new(:time, /\{\{time\}\}/),
    TokenRecognizer.new(:plaintext, /'.*'/),
    TokenRecognizer.new(:plaintext, //)
  ])
end

Public Instance Methods

full_tokenize(pattern) click to toggle source

Tokenizes mixed date and time pattern strings, used to tokenize the additional date format patterns.

# File lib/twitter_cldr/tokenizers/calendars/date_time_tokenizer.rb, line 35
def full_tokenize(pattern)
  PatternTokenizer.new(data_reader, full_tokenizer).tokenize(pattern)
end
tokenize(pattern) click to toggle source
# File lib/twitter_cldr/tokenizers/calendars/date_time_tokenizer.rb, line 27
def tokenize(pattern)
  expand_tokens(
    PatternTokenizer.new(data_reader, tokenizer).tokenize(pattern)
  )
end

Protected Instance Methods

expand_date(token) click to toggle source
# File lib/twitter_cldr/tokenizers/calendars/date_time_tokenizer.rb, line 54
def expand_date(token)
  date_reader = data_reader.date_reader
  date_reader.tokenizer.tokenize(date_reader.pattern)
end
expand_time(token) click to toggle source
# File lib/twitter_cldr/tokenizers/calendars/date_time_tokenizer.rb, line 59
def expand_time(token)
  time_reader = data_reader.time_reader
  time_reader.tokenizer.tokenize(time_reader.pattern)
end
expand_tokens(tokens) click to toggle source
# File lib/twitter_cldr/tokenizers/calendars/date_time_tokenizer.rb, line 41
def expand_tokens(tokens)
  tokens.inject([]) do |ret, token|
    ret + case token.type
      when :date
        expand_date(token)
      when :time
        expand_time(token)
      else
        [token]
    end
  end
end
full_tokenizer() click to toggle source
# File lib/twitter_cldr/tokenizers/calendars/date_time_tokenizer.rb, line 64
def full_tokenizer
  @@full_tokenizer ||= begin
    new_tok = Tokenizer.union(
      data_reader.date_reader.tokenizer.tokenizer,
      data_reader.time_reader.tokenizer.tokenizer
    ) do |recognizer|
      recognizer.token_type != :plaintext
    end

    new_tok.recognizers << TokenRecognizer.new(:plaintext, //)
    new_tok
  end
end
tokenizer() click to toggle source
# File lib/twitter_cldr/tokenizers/calendars/date_time_tokenizer.rb, line 78
def tokenizer
  self.class.tokenizer
end