class Tml::Tokenizers::XMessage

Attributes

label[RW]
last[RW]
len[RW]
options[RW]
pos[RW]
tree[RW]

Public Class Methods

new(text, opts = {}) click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 69
def initialize(text, opts = {})
  @label = text
  @pos = 0
  @len = @label ? @label.length : 0
  @last = nil
  @options = opts || {}
  @tree = nil
  tokenize
end

Public Instance Methods

choice(language, token, token_object) click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 362
def choice(language, token, token_object)
  return unless token

  context_key = token.context_keys.first
  return unless context_key

  ctx = language.context_by_keyword(context_key)
  return unless ctx

  # pp context_key, token_object

  rule = ctx.find_matching_rule(token_object)
  if rule
    # pp context_key, rule.keyword
    return rule_key(context_key, rule.keyword)
  end

  nil
end
choice?(type) click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 392
def choice?(type)
  type == 'choice'
end
collection_format_style(result, c, argument_index, format_type) click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 110
def collection_format_style(result, c, argument_index, format_type)
  # register the format element
  styles = []
  subtype = 'text'; # default

  if c == ','
    # we have a sub-type
    subtype = ''
    c = next_char
    while c && !',}'.index(c)
      subtype += c
      c = next_char
      unless c
        raise "expected ',' or '}', but found end of string"
      end
    end
  end

  result << {index: argument_index, type: format_type, subtype: subtype, styles: styles}

  if c == '}'
    return
  end

  # parse format style
  while c
    c = next_char
    unless c
      raise "expected '}', '|' or format style value, but found end of string"
    end

    if c == '}' && !escaped?
      return
    elsif c == '|'
      next
    end

    style_key = ''
    while c && !'#<|}'.index(c)
      style_key += c
      c = next_char
      unless c
        raise "expected '#', '<' or '|', but found end of string"
      end
    end

    if c == '<'
      style_key += c
    end

    items = []
    styles << {key: style_key, items: items}

    if '#<'.index(c)
      traverse_text(items)
    elsif '|}'.index(c)
      # we found a key without value e.g. {0,param,possessive} and {0,param,prefix#.|possessive}
      revert
    end
  end
end
compile(language, exp, buffer, params) click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 400
def compile(language, exp, buffer, params)
  style = nil

  exp.each do |el|
    token = token_by_type(el[:type], el)
    token_object = get_token_object(params, token)
    token_value = get_token_value(token_object, token, language)

    if el[:styles]
      if choice?(el[:type])
        key = choice(language, token, token_object)
        style = el[:styles].find{ |style|
          style[:key] == key
        }
        if style
          compile(language, style[:items], buffer, params)
        end
      elsif map?(el[:type])
        style = el[:styles].find{ |style|
          style[:key] == token_value
        }
        compile(language, style[:items], buffer, params)
      elsif decoration?(el[:type])
        buffer << token.open_tag(token_object)
        compile(language, el[:styles][0][:items], buffer, params)
        buffer << token.close_tag
      else
        compile(language, el[:styles][0][:items], buffer, params)
      end
    elsif data?(el[:type])
      buffer << token_value
    else
      buffer << el[:value]
    end
  end

  buffer
end
data?(type) click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 382
def data?(type)
  return false unless type
  %w(param number).include?(type)
end
decoration?(type) click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 387
def decoration?(type)
  return false unless type
  Tml.config.xmessage_decoration_tokens.include?(type.to_sym)
end
default_format_style(result, c, argument_index, format_type) click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 195
def default_format_style(result, c, argument_index, format_type)
  # register the format element
  styles = []
  result << {index: argument_index, type: format_type, styles: styles}

  # parse format style
  while c
    c = next_char
    unless c
      raise "expected '}', '|' or format style value, but found end of string"
    end

    if c == '}' && !escaped?
      return
    elsif c == '|'
      next
    end

    style_key = ''
    while c && !'#<+|}'.index(c)
      style_key += c
      c = next_char
      unless c
        raise "expected '#', '<', '+' or '|', but found end of string"
      end
    end

    if c == '<' || c == '+'
      style_key += c
    end

    items = []
    styles << {key: style_key, items: items}

    if '#<+'.index(c)
      traverse_text(items)
    elsif '|}'.index(c)
      # we found a key without value e.g. {0,param,possessive} and {0,param,prefix#.|possessive}
      revert
    end
  end
end
escaped?() click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 102
def escaped?
  @last && @last == '\\'
end
extract_tokens(tree, tokens) click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 461
def extract_tokens(tree, tokens)
  tree.each do |fragment|
    token = token_by_type(fragment[:type], fragment)
    tokens << token if token
    if fragment[:items]
      extract_tokens(fragment[:items], tokens)
    elsif fragment[:styles]
      extract_tokens(fragment[:styles], tokens)
    end
  end
end
get_token_object(token_values, token) click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 473
def get_token_object(token_values, token)
  return nil unless token
  token.token_object(token_values)
end
get_token_value(token_object, token, language) click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 478
def get_token_value(token_object, token, language)
  return nil unless token_object && token
  token.token_value(token_object, language)
end
map?(type) click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 396
def map?(type)
  type == 'map'
end
next_char() click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 83
def next_char
  return if @len == 0 || @pos >= @len
  update_last
  @pos += 1
  @label[@pos - 1]
end
no_format_style(result, c, argument_index, format_type) click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 106
def no_format_style(result, c, argument_index, format_type)
  raise "no format style allowed for format type '" + format_type + "'";
end
optional_style_format_types() click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 56
def optional_style_format_types
  @optional_style_format_types ||= {
      'text' => true,
      'date' => true,
      'time' => true,
      'number' => true,
      'name' => true,
      'list' => true,
      'possessive' => true,
      'salutation' => true
  }
end
peek_char() click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 90
def peek_char
  return if @len == 0
  @label[@pos]
end
revert() click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 95
def revert
  if (@pos > 0)
    @pos -= 1
    update_last
  end
end
rule_key(context_key, rule_key) click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 357
def rule_key(context_key, rule_key)
  return rule_key unless Tml.config.xmessage_rule_key_mapping[context_key.to_sym]
  Tml.config.xmessage_rule_key_mapping[context_key.to_sym][rule_key.to_sym] || rule_key
end
substitute(language, tokens = {}, options = {}) click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 483
def substitute(language, tokens = {}, options = {})
  return @label unless tree
  compile(language, tree, [], tokens).join('')
end
text_format_style(result, c, argument_index, format_type) click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 172
def text_format_style(result, c, argument_index, format_type)
  # parse format style
  buffer = ''
  c = next_char
  unless c
    raise "expected format style or '}', but found end of string"
  end

  while c
    if c == '}'
      result << {index: argument_index, type: format_type, value: buffer}
      return
    end

    # keep adding to buffer
    buffer += c
    c = next_char
    unless c
      raise "expected '}', but found end of string"
    end
  end
end
token_by_type(type, data) click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 447
def token_by_type(type, data)
  if decoration?(type)
    Tml::Tokens::XMessage::Decoration.new(label, data)
  elsif data?(type)
    Tml::Tokens::XMessage::Data.new(label, data)
  elsif choice?(type)
    Tml::Tokens::XMessage::Choice.new(label, data)
  elsif map?(type)
    return Tml::Tokens::XMessage::Map.new(label, data)
  else
    nil
  end
end
tokenize() click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 347
def tokenize
  result = []
  traverse_text(result)
  @tree = result
rescue Exception => ex
  pp ex
  pp "Failed to parse the expression: " + @label
  @tree = nil
end
tokens() click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 439
def tokens
  @tokens ||= begin
    tokens = []
    extract_tokens(tree, tokens)
    tokens.uniq{ |t| [t.class.name, t.full_name] }
  end
end
traverse_format_element(result) click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 238
def traverse_format_element(result)
  argument_index = -1
  format_type = nil
  c = next_char

  unless c
    raise 'expected place holder index, but found end of string'
  end

  if c.match(/[\d:]/)
    # process argument index
    is_keyword = c == ':'
    index = ''
    while c && !',}'.index(c)
      index += c
      c = next_char
      unless c
        raise "expected ',' or '}', but found end of string";
      end
    end

    if !is_keyword && !index.match(/\d+/)
      throw "argument index must be numeric: #{index}"
    end

    argument_index = is_keyword ? index : index * 1
  end

  if c != '}'
    # process format type
    format_type = ''
    c = next_char
    unless c
      raise 'expected format type, but found end of string'
    end

    while c && !',}'.index(c) && !escaped?
      format_type += c
      c = next_char
      unless c
        raise "expected ',' or '}', but found end of string"
      end
    end
  end

  if c == '}' && !escaped?
    if format_type && optional_style_format_types[format_type]
      # we found {0,number} or {0,possessive} or {0,salutation}, which are valid expressions
      result << {type: format_type, index: argument_index}
    else
      if format_type
        # we found something like {0,<type>}, which is invalid.
        raise "expected format style for format type '#{format_type}'"
      end

      # push param format element
      result << {type: 'param', index: argument_index}
    end
  elsif c == ','
    processors = {
        list: 'collection_format_style',
        date: 'text_format_style',
        time: 'text_format_style',
        number: 'text_format_style',
        suffix: 'text_format_style',
        possessive: 'no_format_style',
        salutation: 'no_format_style',
        default: 'default_format_style'
    }
    processor = (processors[format_type.to_sym] || processors[:default])
    self.send(processor, result, c, argument_index, format_type)
  else
    raise "expected ',' or '}', but found '#{c}' at position #{@pos}"
  end
end
traverse_text(result) click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 314
def traverse_text(result)
  in_quoted_string = false
  buffer = ''
  c = next_char

  while c do
    if c == "'"
      in_quoted_string = !in_quoted_string
    end

    if !in_quoted_string && c == '{' && !escaped?
      unless buffer.empty?
        result << {type: 'trans', value: buffer}
        buffer = ''
      end
      traverse_format_element(result)
    elsif !in_quoted_string && (c == '|' || c == '}') && !escaped?
      revert
      break
    else
      buffer += c
    end
    c = next_char
  end

  unless buffer.empty?
    result << {type: 'trans', value: buffer}
    buffer = ''
  end

  result
end
update_last() click to toggle source
# File lib/tml/tokenizers/x_message.rb, line 79
def update_last
  @last = @pos > 0 ? @label[@pos - 1] : nil
end