class AdLint::Cc1::Lexer

Public Class Methods

new(pp_src) click to toggle source
Calls superclass method
# File lib/adlint/cc1/lexer.rb, line 41
def initialize(pp_src)
  super(pp_src.pp_tokens)
  @lst_toks = []
  @nxt_tok = nil
  @ordinary_identifiers = OrdinaryIdentifiers.new
  @identifier_translation = true
end

Public Instance Methods

disable_identifier_translation() click to toggle source
# File lib/adlint/cc1/lexer.rb, line 65
def disable_identifier_translation
  @identifier_translation = false
end
enable_identifier_translation() click to toggle source
# File lib/adlint/cc1/lexer.rb, line 61
def enable_identifier_translation
  @identifier_translation = true
end

Private Instance Methods

concat_contiguous_string_literals(tok, lexer_ctxt) click to toggle source
# File lib/adlint/cc1/lexer.rb, line 151
def concat_contiguous_string_literals(tok, lexer_ctxt)
  until lexer_ctxt.content.empty?
    nxt_tok = tokenize(lexer_ctxt)
    if nxt_tok.type == :STRING_LITERAL
      rslt_tok = tok.class.new(tok.type, tok.value.sub(/"\z/, "") +
                               nxt_tok.value.sub(/\AL?"/, ""),
                               tok.location)
      notify_string_literals_concatenated(tok, nxt_tok, rslt_tok)
      return rslt_tok
    else
      @nxt_tok = nxt_tok
      break
    end
  end
  tok
end
create_lexer_context(tok_ary) click to toggle source
# File lib/adlint/cc1/lexer.rb, line 70
def create_lexer_context(tok_ary)
  LexerContext.new(TokensContent.new(tok_ary))
end
notify_string_literals_concatenated(former, latter, rslt_tok) click to toggle source
# File lib/adlint/cc1/lexer.rb, line 258
def notify_string_literals_concatenated(former, latter, rslt_tok)
  on_string_literals_concatenated.invoke(former, latter, rslt_tok)
end
patch_identifier_translation_mode!() click to toggle source
# File lib/adlint/cc1/lexer.rb, line 168
def patch_identifier_translation_mode!
  keys = [:STRUCT, :UNION, :ENUM]
  head_idx = @lst_toks.rindex { |tok| keys.include?(tok.type) }

  if head_idx
    toks = @lst_toks.drop(head_idx + 1)
    case toks.size
    when 1
      patch_identifier_translation_mode1(*toks)
    when 2
      patch_identifier_translation_mode2(*toks)
    end
  end
end
patch_identifier_translation_mode1(lst_tok1) click to toggle source
# File lib/adlint/cc1/lexer.rb, line 183
def patch_identifier_translation_mode1(lst_tok1)
  case lst_tok1.type
  when :IDENTIFIER
    @identifier_translation = false
  when "{"
    @identifier_translation = true
  end
end
patch_identifier_translation_mode2(lst_tok1, lst_tok2) click to toggle source
# File lib/adlint/cc1/lexer.rb, line 192
def patch_identifier_translation_mode2(lst_tok1, lst_tok2)
  if lst_tok1.type == :IDENTIFIER && lst_tok2.type == "{"
    @identifier_translation = true
  end
end
retokenize_constant(pp_tok, lexer_ctxt) click to toggle source
# File lib/adlint/cc1/lexer.rb, line 206
def retokenize_constant(pp_tok, lexer_ctxt)
  # NOTE: For extended bit-access operators.
  return nil if lst_tok = @lst_toks.last and lst_tok.type == :IDENTIFIER

  case pp_tok.value
  when /\AL?'.*'\z/,
       /\A(?:[0-9]*\.[0-9]+|[0-9]+\.)[FL]*\z/i,
       /\A(?:[0-9]*\.[0-9]*E[+-]?[0-9]+|[0-9]+\.?E[+-]?[0-9]+)[FL]*\z/i,
       /\A(?:0x[0-9a-f]+|0b[01]+|[0-9]+)[UL]*\z/i
    pp_tok.class.new(:CONSTANT, pp_tok.value, pp_tok.location)
  else
    nil
  end
end
retokenize_identifier(pp_tok, lexer_ctxt) click to toggle source
# File lib/adlint/cc1/lexer.rb, line 237
def retokenize_identifier(pp_tok, lexer_ctxt)
  if pp_tok.value =~ /\A[a-z_][a-z_0-9]*\z/i
    pp_tok.class.new(:IDENTIFIER, pp_tok.value, pp_tok.location)
  else
    nil
  end
end
retokenize_keyword(pp_tok, lexer_ctxt) click to toggle source
# File lib/adlint/cc1/lexer.rb, line 198
def retokenize_keyword(pp_tok, lexer_ctxt)
  if keyword = Scanner::KEYWORDS[pp_tok.value]
    pp_tok.class.new(keyword, pp_tok.value, pp_tok.location)
  else
    nil
  end
end
retokenize_null_constant(pp_tok, lexer_ctxt) click to toggle source
# File lib/adlint/cc1/lexer.rb, line 229
def retokenize_null_constant(pp_tok, lexer_ctxt)
  if pp_tok.value == "NULL"
    pp_tok.class.new(:NULL, pp_tok.value, pp_tok.location)
  else
    nil
  end
end
retokenize_punctuator(pp_tok, lexer_ctxt) click to toggle source
# File lib/adlint/cc1/lexer.rb, line 245
def retokenize_punctuator(pp_tok, lexer_ctxt)
  case pp_tok.value
  when "{", "}", "(", ")", "[", "]", ";", ",", "::", ":", "?", "||",
       "|=", "|", "&&", "&=", "&", "^=", "^", "==", "=", "!=", "!",
       "<<=", "<=", "<<", "<", ">>=", ">=", ">>", ">", "+=", "++", "+",
       "->*", "->", "-=", "--", "-", "*=", "*", "/=", "/", "%=", "%",
       "...", ".*", ".", "~"
    pp_tok.class.new(pp_tok.value, pp_tok.value, pp_tok.location)
  else
    nil
  end
end
retokenize_string_literal(pp_tok, lexer_ctxt) click to toggle source
# File lib/adlint/cc1/lexer.rb, line 221
def retokenize_string_literal(pp_tok, lexer_ctxt)
  if pp_tok.value =~ /\AL?".*"\z/
    pp_tok.class.new(:STRING_LITERAL, pp_tok.value, pp_tok.location)
  else
    nil
  end
end
tokenize(lexer_ctxt) click to toggle source
# File lib/adlint/cc1/lexer.rb, line 74
def tokenize(lexer_ctxt)
  if @nxt_tok
    tok = @nxt_tok
    @nxt_tok = nil
  else
    until lexer_ctxt.content.empty?
      pp_tok = lexer_ctxt.content.next_token

      if type_hint = pp_tok.type_hint
        tok = pp_tok.class.new(type_hint, pp_tok.value, pp_tok.location)
      else
        tok = retokenize_keyword(pp_tok, lexer_ctxt)        ||
              retokenize_constant(pp_tok, lexer_ctxt)       ||
              retokenize_string_literal(pp_tok, lexer_ctxt) ||
              retokenize_null_constant(pp_tok, lexer_ctxt)  ||
              retokenize_identifier(pp_tok, lexer_ctxt)     ||
              retokenize_punctuator(pp_tok, lexer_ctxt)
      end

      break if tok
    end
  end

  if tok
    case tok.type
    when :IDENTIFIER
      tok = translate_identifier(tok, lexer_ctxt)
    when :STRING_LITERAL
      tok = concat_contiguous_string_literals(tok, lexer_ctxt)
    end
    @lst_toks.shift if @lst_toks.size == 3
    @lst_toks.push(tok)
    patch_identifier_translation_mode!
  end

  tok
end
translate_identifier(tok, lexer_ctxt) click to toggle source
# File lib/adlint/cc1/lexer.rb, line 112
def translate_identifier(tok, lexer_ctxt)
  # NOTE: To translate given IDENTIFIER into TYPEDEF_NAME if needed.

  # NOTE: The ISO C99 standard says;
  #
  # 6.2.3 Name spaces of identifiers
  #
  # 1 If more than one declaration of a particular identifier is visible at
  #   any point in a translation unit, the syntactic context disambiguates
  #   uses that refer to different entities.  Thus, there are separate name
  #   spaces for various categories of identifiers, as follows:
  #   -- label names (disambiguated by the syntax of the label declaration
  #      and use);
  #   -- the tags of structures, unions, and enumerations (disambiguated by
  #      following any of the keywords struct, union, or enum);
  #   -- the members of structures or unions; each structure or union has a
  #      separate name space for its members (disambiguated by the type of
  #      the expression used to access the member via the . or ->
  #      operator);
  #   -- all other identifiers, called ordinary identifiers (declared in
  #      ordinary declarators or as enumeration constants).

  if @identifier_translation
    if tok.type == :IDENTIFIER
      id_type = @ordinary_identifiers.find(tok.value)
      if id_type == :typedef
        unless lst_tok = @lst_toks.last and
            lst_tok.type == :STRUCT || lst_tok.type == :UNION ||
            lst_tok.type == :ENUM   ||
            lst_tok.type == "->"    || lst_tok.type == "."
          tok = tok.class.new(:TYPEDEF_NAME, tok.value, tok.location)
        end
      end
    end
  end

  tok
end