class ANTLR3::TokenScheme
TokenSchemes exist to handle the problem of defining token types as integer values while maintaining meaningful text names for the types. They are dynamically defined modules that map integer values to constants with token-type names.
Constants
- FETCH_KEY
Attributes
types[R]
unused[R]
Public Class Methods
build( *token_names )
click to toggle source
# File lib/antlr3/token.rb, line 539 def self.build( *token_names ) token_names = [ token_names ].flatten! token_names.compact! token_names.uniq! tk_class = Class === token_names.first ? token_names.shift : nil value_maps, names = token_names.partition { |i| Hash === i } new( tk_class ) do for value_map in value_maps define_tokens( value_map ) end for name in names define_token( name ) end end end
new( tk_class = nil, &body )
click to toggle source
Calls superclass method
# File lib/antlr3/token.rb, line 511 def self.new( tk_class = nil, &body ) super() do tk_class ||= Class.new( ::ANTLR3::CommonToken ) self.token_class = tk_class const_set( :TOKEN_NAMES, ::ANTLR3::Constants::BUILT_IN_TOKEN_NAMES.clone ) @types = ::ANTLR3::Constants::BUILT_IN_TOKEN_NAMES.invert @unused = ::ANTLR3::Constants::MIN_TOKEN_TYPE scheme = self define_method( :token_scheme ) { scheme } define_method( :token_names ) { scheme::TOKEN_NAMES } define_method( :token_name ) do |type| begin token_names[ type ] or super rescue NoMethodError ::ANTLR3::CommonToken.token_name( type ) end end module_function :token_name, :token_names include ANTLR3::Constants body and module_eval( &body ) end end
Public Instance Methods
[]( name_or_value )
click to toggle source
# File lib/antlr3/token.rb, line 649 def []( name_or_value ) case name_or_value when Integer then token_names.fetch( name_or_value, nil ) else const_get( name_or_value.to_s ) rescue FETCH_KEY.call( token_names, name_or_value ) end end
built_in_type?( type_value )
click to toggle source
# File lib/antlr3/token.rb, line 638 def built_in_type?( type_value ) Constants::BUILT_IN_TOKEN_NAMES.fetch( type_value, false ) and true end
define_token( name, value = nil )
click to toggle source
# File lib/antlr3/token.rb, line 572 def define_token( name, value = nil ) name = name.to_s if current_value = @types[ name ] # token type has already been defined # raise an error unless value is the same as the current value value ||= current_value unless current_value == value raise NameError.new( "new token type definition ``#{ name } = #{ value }'' conflicts " << "with existing type definition ``#{ name } = #{ current_value }''", name ) end else value ||= @unused if name =~ /^[A-Z]\w*$/ const_set( name, @types[ name ] = value ) else constant = "T__#{ value }" const_set( constant, @types[ constant ] = value ) @types[ name ] = value end register_name( value, name ) unless built_in_type?( value ) end value >= @unused and @unused = value + 1 return self end
define_tokens( token_map = {} )
click to toggle source
# File lib/antlr3/token.rb, line 565 def define_tokens( token_map = {} ) for token_name, token_value in token_map define_token( token_name, token_value ) end return self end
register_name( type_value, name )
click to toggle source
# File lib/antlr3/token.rb, line 614 def register_name( type_value, name ) name = name.to_s.freeze if token_names.has_key?( type_value ) current_name = token_names[ type_value ] current_name == name and return name if current_name == "T__#{ type_value }" # only an anonymous name is registered -- upgrade the name to the full literal name token_names[ type_value ] = name elsif name == "T__#{ type_value }" # ignore name downgrade from literal to anonymous constant return current_name else error = NameError.new( "attempted assignment of token type #{ type_value }" << " to name #{ name } conflicts with existing name #{ current_name }", name ) raise error end else token_names[ type_value ] = name.to_s.freeze end end
register_names( *names )
click to toggle source
# File lib/antlr3/token.rb, line 601 def register_names( *names ) if names.length == 1 and Hash === names.first names.first.each do |value, name| register_name( value, name ) end else names.each_with_index do |name, i| type_value = Constants::MIN_TOKEN_TYPE + i register_name( type_value, name ) end end end
token_class()
click to toggle source
# File lib/antlr3/token.rb, line 656 def token_class self::Token end
token_class=( klass )
click to toggle source
# File lib/antlr3/token.rb, line 660 def token_class=( klass ) Class === klass or raise( TypeError, "token_class must be a Class" ) Util.silence_warnings do klass < self or klass.send( :include, self ) const_set( :Token, klass ) end end
token_defined?( name_or_value )
click to toggle source
# File lib/antlr3/token.rb, line 642 def token_defined?( name_or_value ) case value when Integer then token_names.has_key?( name_or_value ) else const_defined?( name_or_value.to_s ) end end
Private Instance Methods
included( mod )
click to toggle source
Calls superclass method
# File lib/antlr3/token.rb, line 557 def included( mod ) super mod.extend( self ) end