class AdLint::Cc1::Lexer
Public Class Methods
new(pp_src)
click to toggle source
Calls superclass method
# File lib/adlint/cc1/lexer.rb, line 41 def initialize(pp_src) super(pp_src.pp_tokens) @lst_toks = [] @nxt_tok = nil @ordinary_identifiers = OrdinaryIdentifiers.new @identifier_translation = true end
Public Instance Methods
disable_identifier_translation()
click to toggle source
# File lib/adlint/cc1/lexer.rb, line 65 def disable_identifier_translation @identifier_translation = false end
enable_identifier_translation()
click to toggle source
# File lib/adlint/cc1/lexer.rb, line 61 def enable_identifier_translation @identifier_translation = true end
Private Instance Methods
concat_contiguous_string_literals(tok, lexer_ctxt)
click to toggle source
# File lib/adlint/cc1/lexer.rb, line 151 def concat_contiguous_string_literals(tok, lexer_ctxt) until lexer_ctxt.content.empty? nxt_tok = tokenize(lexer_ctxt) if nxt_tok.type == :STRING_LITERAL rslt_tok = tok.class.new(tok.type, tok.value.sub(/"\z/, "") + nxt_tok.value.sub(/\AL?"/, ""), tok.location) notify_string_literals_concatenated(tok, nxt_tok, rslt_tok) return rslt_tok else @nxt_tok = nxt_tok break end end tok end
create_lexer_context(tok_ary)
click to toggle source
# File lib/adlint/cc1/lexer.rb, line 70 def create_lexer_context(tok_ary) LexerContext.new(TokensContent.new(tok_ary)) end
notify_string_literals_concatenated(former, latter, rslt_tok)
click to toggle source
# File lib/adlint/cc1/lexer.rb, line 258 def notify_string_literals_concatenated(former, latter, rslt_tok) on_string_literals_concatenated.invoke(former, latter, rslt_tok) end
patch_identifier_translation_mode!()
click to toggle source
# File lib/adlint/cc1/lexer.rb, line 168 def patch_identifier_translation_mode! keys = [:STRUCT, :UNION, :ENUM] head_idx = @lst_toks.rindex { |tok| keys.include?(tok.type) } if head_idx toks = @lst_toks.drop(head_idx + 1) case toks.size when 1 patch_identifier_translation_mode1(*toks) when 2 patch_identifier_translation_mode2(*toks) end end end
patch_identifier_translation_mode1(lst_tok1)
click to toggle source
# File lib/adlint/cc1/lexer.rb, line 183 def patch_identifier_translation_mode1(lst_tok1) case lst_tok1.type when :IDENTIFIER @identifier_translation = false when "{" @identifier_translation = true end end
patch_identifier_translation_mode2(lst_tok1, lst_tok2)
click to toggle source
# File lib/adlint/cc1/lexer.rb, line 192 def patch_identifier_translation_mode2(lst_tok1, lst_tok2) if lst_tok1.type == :IDENTIFIER && lst_tok2.type == "{" @identifier_translation = true end end
retokenize_constant(pp_tok, lexer_ctxt)
click to toggle source
# File lib/adlint/cc1/lexer.rb, line 206 def retokenize_constant(pp_tok, lexer_ctxt) # NOTE: For extended bit-access operators. return nil if lst_tok = @lst_toks.last and lst_tok.type == :IDENTIFIER case pp_tok.value when /\AL?'.*'\z/, /\A(?:[0-9]*\.[0-9]+|[0-9]+\.)[FL]*\z/i, /\A(?:[0-9]*\.[0-9]*E[+-]?[0-9]+|[0-9]+\.?E[+-]?[0-9]+)[FL]*\z/i, /\A(?:0x[0-9a-f]+|0b[01]+|[0-9]+)[UL]*\z/i pp_tok.class.new(:CONSTANT, pp_tok.value, pp_tok.location) else nil end end
retokenize_identifier(pp_tok, lexer_ctxt)
click to toggle source
# File lib/adlint/cc1/lexer.rb, line 237 def retokenize_identifier(pp_tok, lexer_ctxt) if pp_tok.value =~ /\A[a-z_][a-z_0-9]*\z/i pp_tok.class.new(:IDENTIFIER, pp_tok.value, pp_tok.location) else nil end end
retokenize_keyword(pp_tok, lexer_ctxt)
click to toggle source
# File lib/adlint/cc1/lexer.rb, line 198 def retokenize_keyword(pp_tok, lexer_ctxt) if keyword = Scanner::KEYWORDS[pp_tok.value] pp_tok.class.new(keyword, pp_tok.value, pp_tok.location) else nil end end
retokenize_null_constant(pp_tok, lexer_ctxt)
click to toggle source
# File lib/adlint/cc1/lexer.rb, line 229 def retokenize_null_constant(pp_tok, lexer_ctxt) if pp_tok.value == "NULL" pp_tok.class.new(:NULL, pp_tok.value, pp_tok.location) else nil end end
retokenize_punctuator(pp_tok, lexer_ctxt)
click to toggle source
# File lib/adlint/cc1/lexer.rb, line 245 def retokenize_punctuator(pp_tok, lexer_ctxt) case pp_tok.value when "{", "}", "(", ")", "[", "]", ";", ",", "::", ":", "?", "||", "|=", "|", "&&", "&=", "&", "^=", "^", "==", "=", "!=", "!", "<<=", "<=", "<<", "<", ">>=", ">=", ">>", ">", "+=", "++", "+", "->*", "->", "-=", "--", "-", "*=", "*", "/=", "/", "%=", "%", "...", ".*", ".", "~" pp_tok.class.new(pp_tok.value, pp_tok.value, pp_tok.location) else nil end end
retokenize_string_literal(pp_tok, lexer_ctxt)
click to toggle source
# File lib/adlint/cc1/lexer.rb, line 221 def retokenize_string_literal(pp_tok, lexer_ctxt) if pp_tok.value =~ /\AL?".*"\z/ pp_tok.class.new(:STRING_LITERAL, pp_tok.value, pp_tok.location) else nil end end
tokenize(lexer_ctxt)
click to toggle source
# File lib/adlint/cc1/lexer.rb, line 74 def tokenize(lexer_ctxt) if @nxt_tok tok = @nxt_tok @nxt_tok = nil else until lexer_ctxt.content.empty? pp_tok = lexer_ctxt.content.next_token if type_hint = pp_tok.type_hint tok = pp_tok.class.new(type_hint, pp_tok.value, pp_tok.location) else tok = retokenize_keyword(pp_tok, lexer_ctxt) || retokenize_constant(pp_tok, lexer_ctxt) || retokenize_string_literal(pp_tok, lexer_ctxt) || retokenize_null_constant(pp_tok, lexer_ctxt) || retokenize_identifier(pp_tok, lexer_ctxt) || retokenize_punctuator(pp_tok, lexer_ctxt) end break if tok end end if tok case tok.type when :IDENTIFIER tok = translate_identifier(tok, lexer_ctxt) when :STRING_LITERAL tok = concat_contiguous_string_literals(tok, lexer_ctxt) end @lst_toks.shift if @lst_toks.size == 3 @lst_toks.push(tok) patch_identifier_translation_mode! end tok end
translate_identifier(tok, lexer_ctxt)
click to toggle source
# File lib/adlint/cc1/lexer.rb, line 112 def translate_identifier(tok, lexer_ctxt) # NOTE: To translate given IDENTIFIER into TYPEDEF_NAME if needed. # NOTE: The ISO C99 standard says; # # 6.2.3 Name spaces of identifiers # # 1 If more than one declaration of a particular identifier is visible at # any point in a translation unit, the syntactic context disambiguates # uses that refer to different entities. Thus, there are separate name # spaces for various categories of identifiers, as follows: # -- label names (disambiguated by the syntax of the label declaration # and use); # -- the tags of structures, unions, and enumerations (disambiguated by # following any of the keywords struct, union, or enum); # -- the members of structures or unions; each structure or union has a # separate name space for its members (disambiguated by the type of # the expression used to access the member via the . or -> # operator); # -- all other identifiers, called ordinary identifiers (declared in # ordinary declarators or as enumeration constants). if @identifier_translation if tok.type == :IDENTIFIER id_type = @ordinary_identifiers.find(tok.value) if id_type == :typedef unless lst_tok = @lst_toks.last and lst_tok.type == :STRUCT || lst_tok.type == :UNION || lst_tok.type == :ENUM || lst_tok.type == "->" || lst_tok.type == "." tok = tok.class.new(:TYPEDEF_NAME, tok.value, tok.location) end end end end tok end