class ANTLR3::TokenScheme

TokenSchemes exist to handle the problem of defining token types as integer values while maintaining meaningful text names for the types. They are dynamically defined modules that map integer values to constants with token-type names.

Constants

FETCH_KEY

Attributes

types[R]
unused[R]

Public Class Methods

build( *token_names ) click to toggle source
# File lib/antlr3/token.rb, line 539
def self.build( *token_names )
  token_names = [ token_names ].flatten!
  token_names.compact!
  token_names.uniq!
  tk_class = Class === token_names.first ? token_names.shift : nil
  value_maps, names = token_names.partition { |i| Hash === i }
  new( tk_class ) do
    for value_map in value_maps
      define_tokens( value_map )
    end
    
    for name in names
      define_token( name )
    end
  end
end
new( tk_class = nil, &body ) click to toggle source
Calls superclass method
# File lib/antlr3/token.rb, line 511
def self.new( tk_class = nil, &body )
  super() do
    tk_class ||= Class.new( ::ANTLR3::CommonToken )
    self.token_class = tk_class
    
    const_set( :TOKEN_NAMES, ::ANTLR3::Constants::BUILT_IN_TOKEN_NAMES.clone )
    
    @types  = ::ANTLR3::Constants::BUILT_IN_TOKEN_NAMES.invert
    @unused = ::ANTLR3::Constants::MIN_TOKEN_TYPE
    
    scheme = self
    define_method( :token_scheme ) { scheme }
    define_method( :token_names )  { scheme::TOKEN_NAMES }
    define_method( :token_name ) do |type|
      begin
        token_names[ type ] or super
      rescue NoMethodError
        ::ANTLR3::CommonToken.token_name( type )
      end
    end
    module_function :token_name, :token_names
    
    include ANTLR3::Constants
    
    body and module_eval( &body )
  end
end

Public Instance Methods

[]( name_or_value ) click to toggle source
# File lib/antlr3/token.rb, line 649
def []( name_or_value )
  case name_or_value
  when Integer then token_names.fetch( name_or_value, nil )
  else const_get( name_or_value.to_s ) rescue FETCH_KEY.call( token_names, name_or_value )
  end
end
built_in_type?( type_value ) click to toggle source
# File lib/antlr3/token.rb, line 638
def built_in_type?( type_value )
  Constants::BUILT_IN_TOKEN_NAMES.fetch( type_value, false ) and true
end
define_token( name, value = nil ) click to toggle source
# File lib/antlr3/token.rb, line 572
def define_token( name, value = nil )
  name = name.to_s
  
  if current_value = @types[ name ]
    # token type has already been defined
    # raise an error unless value is the same as the current value
    value ||= current_value
    unless current_value == value
      raise NameError.new( 
        "new token type definition ``#{ name } = #{ value }'' conflicts " <<
        "with existing type definition ``#{ name } = #{ current_value }''", name
      )
    end
  else
    value ||= @unused
    if name =~ /^[A-Z]\w*$/
      const_set( name, @types[ name ] = value )
    else
      constant = "T__#{ value }"
      const_set( constant, @types[ constant ] = value )
      @types[ name ] = value
    end
    register_name( value, name ) unless built_in_type?( value )
  end
  
  value >= @unused and @unused = value + 1
  return self
end
define_tokens( token_map = {} ) click to toggle source
# File lib/antlr3/token.rb, line 565
def define_tokens( token_map = {} )
  for token_name, token_value in token_map
    define_token( token_name, token_value )
  end
  return self
end
register_name( type_value, name ) click to toggle source
# File lib/antlr3/token.rb, line 614
def register_name( type_value, name )
  name = name.to_s.freeze
  if token_names.has_key?( type_value )
    current_name = token_names[ type_value ]
    current_name == name and return name
    
    if current_name == "T__#{ type_value }"
      # only an anonymous name is registered -- upgrade the name to the full literal name
      token_names[ type_value ] = name
    elsif name == "T__#{ type_value }"
      # ignore name downgrade from literal to anonymous constant
      return current_name
    else
      error = NameError.new( 
        "attempted assignment of token type #{ type_value }" <<
        " to name #{ name } conflicts with existing name #{ current_name }", name
      )
      raise error
    end
  else
    token_names[ type_value ] = name.to_s.freeze
  end
end
register_names( *names ) click to toggle source
# File lib/antlr3/token.rb, line 601
def register_names( *names )
  if names.length == 1 and Hash === names.first
    names.first.each do |value, name|
      register_name( value, name )
    end
  else
    names.each_with_index do |name, i|
      type_value = Constants::MIN_TOKEN_TYPE + i
      register_name( type_value, name )
    end
  end
end
token_class() click to toggle source
# File lib/antlr3/token.rb, line 656
def token_class
  self::Token
end
token_class=( klass ) click to toggle source
# File lib/antlr3/token.rb, line 660
def token_class=( klass )
  Class === klass or raise( TypeError, "token_class must be a Class" )
  Util.silence_warnings do
    klass < self or klass.send( :include, self )
    const_set( :Token, klass )
  end
end
token_defined?( name_or_value ) click to toggle source
# File lib/antlr3/token.rb, line 642
def token_defined?( name_or_value )
  case value
  when Integer then token_names.has_key?( name_or_value )
  else const_defined?( name_or_value.to_s )
  end
end

Private Instance Methods

included( mod ) click to toggle source
Calls superclass method
# File lib/antlr3/token.rb, line 557
def included( mod )
  super
  mod.extend( self )
end