ANTLR3C  3.3.1
antlr3lexer.c File Reference

Base implementation of an antlr 3 lexer. More...

#include <antlr3lexer.h>
Include dependency graph for antlr3lexer.c:

Functions

ANTLR3_API pANTLR3_LEXER antlr3LexerNew (ANTLR3_UINT32 sizeHint, pANTLR3_RECOGNIZER_SHARED_STATE state)
 
ANTLR3_API pANTLR3_LEXER antlr3LexerNewStream (ANTLR3_UINT32 sizeHint, pANTLR3_INPUT_STREAM input, pANTLR3_RECOGNIZER_SHARED_STATE state)
 
static void displayRecognitionError (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_UINT8 *tokenNames)
 Default lexer error handler (works for 8 bit streams only!!!) More...
 
static pANTLR3_COMMON_TOKEN emit (pANTLR3_LEXER lexer)
 
static void emitNew (pANTLR3_LEXER lexer, pANTLR3_COMMON_TOKEN token)
 
static void freeLexer (pANTLR3_LEXER lexer)
 Free the resources allocated by a lexer. More...
 
static ANTLR3_MARKER getCharIndex (pANTLR3_LEXER lexer)
 
static ANTLR3_UINT32 getCharPositionInLine (pANTLR3_LEXER lexer)
 
static void * getCurrentInputSymbol (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM istream)
 
static ANTLR3_UINT32 getLine (pANTLR3_LEXER lexer)
 
static void * getMissingSymbol (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM istream, pANTLR3_EXCEPTION e, ANTLR3_UINT32 expectedTokenType, pANTLR3_BITSET_LIST follow)
 
static pANTLR3_STRING getText (pANTLR3_LEXER lexer)
 
static void matchAny (pANTLR3_LEXER lexer)
 
static ANTLR3_BOOLEAN matchc (pANTLR3_LEXER lexer, ANTLR3_UCHAR c)
 Implementation of matchc for the lexer, overrides any base implementation in the base recognizer. More...
 
static ANTLR3_BOOLEAN matchRange (pANTLR3_LEXER lexer, ANTLR3_UCHAR low, ANTLR3_UCHAR high)
 Implementation of match range for the lexer, overrides any base implementation in the base recognizer. More...
 
static ANTLR3_BOOLEAN matchs (pANTLR3_LEXER lexer, ANTLR3_UCHAR *string)
 Implementation of matchs for the lexer, overrides any base implementation in the base recognizer. More...
 
static void mTokens (pANTLR3_LEXER lexer)
 
static pANTLR3_COMMON_TOKEN nextToken (pANTLR3_TOKEN_SOURCE toksource)
 Default implementation of the nextToken() call for a lexer. More...
 
static ANTLR3_INLINE pANTLR3_COMMON_TOKEN nextTokenStr (pANTLR3_TOKEN_SOURCE toksource)
 Returns the next available token from the current input stream. More...
 
static void popCharStream (pANTLR3_LEXER lexer)
 Stops using the current input stream and reverts to any prior input stream on the stack. More...
 
static void pushCharStream (pANTLR3_LEXER lexer, pANTLR3_INPUT_STREAM input)
 Change to a new input stream, remembering the old one. More...
 
static void recover (pANTLR3_LEXER lexer)
 
static void reportError (pANTLR3_BASE_RECOGNIZER rec)
 
static void reset (pANTLR3_BASE_RECOGNIZER rec)
 
static void setCharStream (pANTLR3_LEXER lexer, pANTLR3_INPUT_STREAM input)
 

Detailed Description

Base implementation of an antlr 3 lexer.

An ANTLR3 lexer implements a base recongizer, a token source and a lexer interface. It constructs a base recognizer with default functions, then overrides any of these that are parser specific (usual default implementation of base recognizer.

Function Documentation

◆ antlr3LexerNew()

ANTLR3_API pANTLR3_LEXER antlr3LexerNew ( ANTLR3_UINT32  sizeHint,
pANTLR3_RECOGNIZER_SHARED_STATE  state 
)

References ANTLR3_CALLOC, ANTLR3_MALLOC, ANTLR3_TEXT_NONE, ANTLR3_TOKEN_EOF, ANTLR3_TOKEN_INVALID, ANTLR3_TRUE, ANTLR3_TYPE_LEXER, antlr3BaseRecognizerNew(), antlr3SetTokenAPI(), ANTLR3_COMMON_TOKEN_struct::custom, displayRecognitionError(), ANTLR3_BASE_RECOGNIZER_struct::displayRecognitionError, emit(), ANTLR3_LEXER_struct::emit, emitNew(), ANTLR3_LEXER_struct::emitNew, ANTLR3_TOKEN_SOURCE_struct::eofToken, ANTLR3_COMMON_TOKEN_struct::factoryMade, ANTLR3_BASE_RECOGNIZER_struct::free, ANTLR3_LEXER_struct::free, freeLexer(), getCharIndex(), ANTLR3_LEXER_struct::getCharIndex, getCharPositionInLine(), ANTLR3_LEXER_struct::getCharPositionInLine, getCurrentInputSymbol(), ANTLR3_BASE_RECOGNIZER_struct::getCurrentInputSymbol, getLine(), ANTLR3_LEXER_struct::getLine, getMissingSymbol(), ANTLR3_BASE_RECOGNIZER_struct::getMissingSymbol, getText(), ANTLR3_LEXER_struct::getText, matchAny(), ANTLR3_LEXER_struct::matchAny, matchc(), ANTLR3_LEXER_struct::matchc, matchRange(), ANTLR3_LEXER_struct::matchRange, matchs(), ANTLR3_LEXER_struct::matchs, mTokens(), ANTLR3_LEXER_struct::mTokens, nextToken(), ANTLR3_TOKEN_SOURCE_struct::nextToken, popCharStream(), ANTLR3_LEXER_struct::popCharStream, pushCharStream(), ANTLR3_LEXER_struct::pushCharStream, ANTLR3_LEXER_struct::rec, recover(), ANTLR3_LEXER_struct::recover, reportError(), ANTLR3_BASE_RECOGNIZER_struct::reportError, reset(), ANTLR3_BASE_RECOGNIZER_struct::reset, setCharStream(), ANTLR3_LEXER_struct::setCharStream, ANTLR3_COMMON_TOKEN_struct::setType, ANTLR3_TOKEN_SOURCE_struct::skipToken, ANTLR3_BASE_RECOGNIZER_struct::state, ANTLR3_COMMON_TOKEN_struct::strFactory, ANTLR3_TOKEN_SOURCE_struct::strFactory, ANTLR3_BASE_RECOGNIZER_struct::super, ANTLR3_TOKEN_SOURCE_struct::super, ANTLR3_COMMON_TOKEN_struct::textState, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokFactory, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokSource, ANTLR3_COMMON_TOKEN_struct::user1, ANTLR3_COMMON_TOKEN_struct::user2, and ANTLR3_COMMON_TOKEN_struct::user3.

Referenced by antlr3LexerNewStream().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ antlr3LexerNewStream()

ANTLR3_API pANTLR3_LEXER antlr3LexerNewStream ( ANTLR3_UINT32  sizeHint,
pANTLR3_INPUT_STREAM  input,
pANTLR3_RECOGNIZER_SHARED_STATE  state 
)

References antlr3LexerNew(), and setCharStream().

Here is the call graph for this function:

◆ displayRecognitionError()

◆ emit()

static pANTLR3_COMMON_TOKEN emit ( pANTLR3_LEXER  lexer)
static

◆ emitNew()

static void emitNew ( pANTLR3_LEXER  lexer,
pANTLR3_COMMON_TOKEN  token 
)
static

References ANTLR3_LEXER_struct::rec, ANTLR3_BASE_RECOGNIZER_struct::state, and ANTLR3_RECOGNIZER_SHARED_STATE_struct::token.

Referenced by antlr3LexerNew().

Here is the caller graph for this function:

◆ freeLexer()

◆ getCharIndex()

static ANTLR3_MARKER getCharIndex ( pANTLR3_LEXER  lexer)
static

References ANTLR3_INT_STREAM_struct::index, ANTLR3_LEXER_struct::input, and ANTLR3_INPUT_STREAM_struct::istream.

Referenced by antlr3LexerNew().

Here is the caller graph for this function:

◆ getCharPositionInLine()

static ANTLR3_UINT32 getCharPositionInLine ( pANTLR3_LEXER  lexer)
static

References ANTLR3_INPUT_STREAM_struct::charPositionInLine, and ANTLR3_LEXER_struct::input.

Referenced by antlr3LexerNew().

Here is the caller graph for this function:

◆ getCurrentInputSymbol()

static void * getCurrentInputSymbol ( pANTLR3_BASE_RECOGNIZER  recognizer,
pANTLR3_INT_STREAM  istream 
)
static

Referenced by antlr3LexerNew().

Here is the caller graph for this function:

◆ getLine()

static ANTLR3_UINT32 getLine ( pANTLR3_LEXER  lexer)
static

References ANTLR3_INPUT_STREAM_struct::getLine, and ANTLR3_LEXER_struct::input.

Referenced by antlr3LexerNew().

Here is the caller graph for this function:

◆ getMissingSymbol()

static void * getMissingSymbol ( pANTLR3_BASE_RECOGNIZER  recognizer,
pANTLR3_INT_STREAM  istream,
pANTLR3_EXCEPTION  e,
ANTLR3_UINT32  expectedTokenType,
pANTLR3_BITSET_LIST  follow 
)
static

Referenced by antlr3LexerNew().

Here is the caller graph for this function:

◆ getText()

◆ matchAny()

static void matchAny ( pANTLR3_LEXER  lexer)
static

References ANTLR3_INT_STREAM_struct::consume, ANTLR3_LEXER_struct::input, and ANTLR3_INPUT_STREAM_struct::istream.

Referenced by antlr3LexerNew().

Here is the caller graph for this function:

◆ matchc()

static ANTLR3_BOOLEAN matchc ( pANTLR3_LEXER  lexer,
ANTLR3_UCHAR  c 
)
static

Implementation of matchc for the lexer, overrides any base implementation in the base recognizer.

Remarks
Note that the generated code lays down arrays of ints for constant strings so that they are int UTF32 form!

References ANTLR3_INT_STREAM_struct::_LA, ANTLR3_FALSE, ANTLR3_TRUE, ANTLR3_RECOGNIZER_SHARED_STATE_struct::backtracking, ANTLR3_INT_STREAM_struct::consume, ANTLR3_BASE_RECOGNIZER_struct::exConstruct, ANTLR3_RECOGNIZER_SHARED_STATE_struct::failed, ANTLR3_LEXER_struct::input, ANTLR3_INPUT_STREAM_struct::istream, ANTLR3_LEXER_struct::rec, ANTLR3_LEXER_struct::recover, and ANTLR3_BASE_RECOGNIZER_struct::state.

Referenced by antlr3LexerNew().

Here is the caller graph for this function:

◆ matchRange()

static ANTLR3_BOOLEAN matchRange ( pANTLR3_LEXER  lexer,
ANTLR3_UCHAR  low,
ANTLR3_UCHAR  high 
)
static

Implementation of match range for the lexer, overrides any base implementation in the base recognizer.

Remarks
Note that the generated code lays down arrays of ints for constant strings so that they are int UTF32 form!

References ANTLR3_INT_STREAM_struct::_LA, ANTLR3_FALSE, ANTLR3_TRUE, ANTLR3_RECOGNIZER_SHARED_STATE_struct::backtracking, ANTLR3_INT_STREAM_struct::consume, ANTLR3_BASE_RECOGNIZER_struct::exConstruct, ANTLR3_RECOGNIZER_SHARED_STATE_struct::failed, ANTLR3_LEXER_struct::input, ANTLR3_INPUT_STREAM_struct::istream, ANTLR3_LEXER_struct::rec, ANTLR3_LEXER_struct::recover, and ANTLR3_BASE_RECOGNIZER_struct::state.

Referenced by antlr3LexerNew().

Here is the caller graph for this function:

◆ matchs()

static ANTLR3_BOOLEAN matchs ( pANTLR3_LEXER  lexer,
ANTLR3_UCHAR string 
)
static

Implementation of matchs for the lexer, overrides any base implementation in the base recognizer.

Remarks
Note that the generated code lays down arrays of ints for constant strings so that they are int UTF32 form!

References ANTLR3_INT_STREAM_struct::_LA, ANTLR3_FALSE, ANTLR3_STRING_TERMINATOR, ANTLR3_TRUE, ANTLR3_RECOGNIZER_SHARED_STATE_struct::backtracking, ANTLR3_INT_STREAM_struct::consume, ANTLR3_BASE_RECOGNIZER_struct::exConstruct, ANTLR3_RECOGNIZER_SHARED_STATE_struct::failed, ANTLR3_LEXER_struct::input, ANTLR3_INPUT_STREAM_struct::istream, ANTLR3_LEXER_struct::rec, ANTLR3_LEXER_struct::recover, and ANTLR3_BASE_RECOGNIZER_struct::state.

Referenced by antlr3LexerNew().

Here is the caller graph for this function:

◆ mTokens()

static void mTokens ( pANTLR3_LEXER  lexer)
static

References ANTLR3_FPRINTF.

Referenced by antlr3LexerNew().

Here is the caller graph for this function:

◆ nextToken()

static pANTLR3_COMMON_TOKEN nextToken ( pANTLR3_TOKEN_SOURCE  toksource)
static

Default implementation of the nextToken() call for a lexer.

Parameters
toksourcePoints to the implementation of a token source. The lexer is addressed by the super structure pointer.
Returns
The next token in the current input stream or the EOF token if there are no more tokens in any input stream in the stack.

Write detailed description for nextToken here.

Remarks
Write remarks for nextToken here.
See also
nextTokenStr

References ANTLR3_TOKEN_EOF, nextTokenStr(), ANTLR3_LEXER_struct::popCharStream, ANTLR3_LEXER_struct::rec, ANTLR3_STACK_struct::size, ANTLR3_BASE_RECOGNIZER_struct::state, ANTLR3_RECOGNIZER_SHARED_STATE_struct::streams, ANTLR3_TOKEN_SOURCE_struct::super, and ANTLR3_COMMON_TOKEN_struct::type.

Referenced by antlr3LexerNew().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ nextTokenStr()

static ANTLR3_INLINE pANTLR3_COMMON_TOKEN nextTokenStr ( pANTLR3_TOKEN_SOURCE  toksource)
static

Returns the next available token from the current input stream.

Parameters
toksourcePoints to the implementation of a token source. The lexer is addressed by the super structure pointer.
Returns
The next token in the current input stream or the EOF token if there are no more tokens.
Remarks
Write remarks for nextToken here.
See also
nextToken

Loop until we get a non skipped token or EOF

References ANTLR3_INT_STREAM_struct::_LA, ANTLR3_CHARSTREAM_EOF, ANTLR3_FALSE, ANTLR3_TOKEN_DEFAULT_CHANNEL, ANTLR3_TRUE, ANTLR3_RECOGNIZER_SHARED_STATE_struct::channel, ANTLR3_INPUT_STREAM_struct::charPositionInLine, ANTLR3_LEXER_struct::ctx, ANTLR3_RECOGNIZER_SHARED_STATE_struct::custom, emit(), ANTLR3_TOKEN_SOURCE_struct::eofToken, ANTLR3_RECOGNIZER_SHARED_STATE_struct::error, ANTLR3_COMMON_TOKEN_struct::factoryMade, ANTLR3_RECOGNIZER_SHARED_STATE_struct::failed, ANTLR3_LEXER_struct::getCharIndex, ANTLR3_LEXER_struct::getLine, ANTLR3_LEXER_struct::input, ANTLR3_INPUT_STREAM_struct::istream, ANTLR3_INPUT_STREAM_struct::line, ANTLR3_LEXER_struct::mTokens, ANTLR3_INPUT_STREAM_struct::nextChar, ANTLR3_LEXER_struct::rec, ANTLR3_LEXER_struct::recover, ANTLR3_BASE_RECOGNIZER_struct::reportError, ANTLR3_COMMON_TOKEN_struct::setLine, ANTLR3_COMMON_TOKEN_struct::setStartIndex, ANTLR3_COMMON_TOKEN_struct::setStopIndex, ANTLR3_TOKEN_SOURCE_struct::skipToken, ANTLR3_BASE_RECOGNIZER_struct::state, ANTLR3_TOKEN_SOURCE_struct::super, ANTLR3_RECOGNIZER_SHARED_STATE_struct::text, ANTLR3_RECOGNIZER_SHARED_STATE_struct::token, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokenStartCharIndex, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokenStartCharPositionInLine, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokenStartLine, ANTLR3_RECOGNIZER_SHARED_STATE_struct::user1, ANTLR3_RECOGNIZER_SHARED_STATE_struct::user2, and ANTLR3_RECOGNIZER_SHARED_STATE_struct::user3.

Referenced by nextToken().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ popCharStream()

static void popCharStream ( pANTLR3_LEXER  lexer)
static

Stops using the current input stream and reverts to any prior input stream on the stack.

Parameters
lexerDescription of parameter lexer.

Pointer to a function that abandons the current input stream, whether it is empty or not and reverts to the previous stacked input stream.

Remarks
The function fails silently if there are no prior input streams.

References ANTLR3_LEXER_struct::input, ANTLR3_INPUT_STREAM_struct::istream, ANTLR3_STACK_struct::pop, ANTLR3_LEXER_struct::rec, ANTLR3_INT_STREAM_struct::rewindLast, ANTLR3_LEXER_struct::setCharStream, ANTLR3_STACK_struct::size, ANTLR3_BASE_RECOGNIZER_struct::state, ANTLR3_RECOGNIZER_SHARED_STATE_struct::streams, and ANTLR3_STACK_struct::top.

Referenced by antlr3LexerNew().

Here is the caller graph for this function:

◆ pushCharStream()

static void pushCharStream ( pANTLR3_LEXER  lexer,
pANTLR3_INPUT_STREAM  input 
)
static

Change to a new input stream, remembering the old one.

Parameters
lexerPointer to the lexer instance to switch input streams for.
inputNew input stream to install as the current one.

Switches the current character input stream to a new one, saving the old one, which we will revert to at the end of this new one.

References antlr3StackNew(), ANTLR3_LEXER_struct::input, ANTLR3_INPUT_STREAM_struct::istream, ANTLR3_INT_STREAM_struct::mark, ANTLR3_STACK_struct::push, ANTLR3_LEXER_struct::rec, ANTLR3_LEXER_struct::setCharStream, ANTLR3_BASE_RECOGNIZER_struct::state, and ANTLR3_RECOGNIZER_SHARED_STATE_struct::streams.

Referenced by antlr3LexerNew().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ recover()

static void recover ( pANTLR3_LEXER  lexer)
static

References ANTLR3_INT_STREAM_struct::consume, ANTLR3_LEXER_struct::input, and ANTLR3_INPUT_STREAM_struct::istream.

Referenced by antlr3LexerNew().

Here is the caller graph for this function:

◆ reportError()

◆ reset()

◆ setCharStream()