ANTLR3C  3.3.1
antlr3tokenstream.c File Reference

Default implementation of CommonTokenStream. More...

Include dependency graph for antlr3tokenstream.c:

Functions

static ANTLR3_UINT32 _LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
 
ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM antlr3CommonTokenDebugStreamSourceNew (ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger)
 
ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM antlr3CommonTokenStreamNew (ANTLR3_UINT32 hint)
 
ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM antlr3CommonTokenStreamSourceNew (ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source)
 
static void antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream)
 
static void antlr3TokenStreamFree (pANTLR3_TOKEN_STREAM stream)
 
ANTLR3_API pANTLR3_TOKEN_STREAM antlr3TokenStreamNew ()
 
static void consume (pANTLR3_INT_STREAM is)
 Move the input pointer to the next incoming token. More...
 
static void consumeInitialHiddenTokens (pANTLR3_INT_STREAM is)
 Debug only method to flag consumption of initial off-channel tokens in the input stream. More...
 
static void dbgConsume (pANTLR3_INT_STREAM is)
 As per ordinary consume but notifies the debugger about hidden tokens and so on. More...
 
static ANTLR3_UINT32 dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
 As per _LA() but for debug mode. More...
 
static ANTLR3_MARKER dbgMark (pANTLR3_INT_STREAM is)
 As per mark() but with a call to tell the debugger we are doing this. More...
 
static void dbgRewindLast (pANTLR3_INT_STREAM is)
 
static void dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
 
static void dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
 
static pANTLR3_COMMON_TOKEN dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
 As per the normal tokLT but sends information to the debugger. More...
 
static void discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard)
 
static void discardTokenType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype)
 
static void fillBuffer (pANTLR3_COMMON_TOKEN_STREAM tokenStream)
 
ANTLR3_API void fillBufferExt (pANTLR3_COMMON_TOKEN_STREAM tokenStream)
 
static pANTLR3_COMMON_TOKEN get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i)
 
static pANTLR3_STRING getSourceName (pANTLR3_INT_STREAM is)
 Return a string that represents the name assoicated with the input source. More...
 
static pANTLR3_LIST getTokenRange (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
 
static pANTLR3_VECTOR getTokens (pANTLR3_COMMON_TOKEN_STREAM cts)
 
static pANTLR3_LIST getTokensList (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list)
 
static pANTLR3_TOKEN_SOURCE getTokenSource (pANTLR3_TOKEN_STREAM ts)
 
static pANTLR3_LIST getTokensSet (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types)
 Given a start and stop index, return a List of all tokens in the token type BitSet. More...
 
static pANTLR3_LIST getTokensType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type)
 
static pANTLR3_COMMON_TOKEN LB (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i)
 
static ANTLR3_MARKER mark (pANTLR3_INT_STREAM is)
 
static void release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark)
 
static void reset (pANTLR3_COMMON_TOKEN_STREAM cts)
 
static void rewindLast (pANTLR3_INT_STREAM is)
 
static void rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
 
static void seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
 
static void setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger)
 
static void setTokenSource (pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource)
 
static void setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel)
 A simple filter mechanism whereby you can tell this token stream to force all tokens of type ttype to be on channel. More...
 
static ANTLR3_UINT32 size (pANTLR3_INT_STREAM is)
 
static ANTLR3_UINT32 skipOffTokenChannels (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i)
 Given a starting index, return the index of the first on-channel token. More...
 
static ANTLR3_UINT32 skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i)
 
static ANTLR3_MARKER tindex (pANTLR3_INT_STREAM is)
 
static pANTLR3_COMMON_TOKEN tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
 Get the ith token from the current position 1..n where k=1 is the first symbol of lookahead. More...
 
static pANTLR3_STRING toString (pANTLR3_TOKEN_STREAM ts)
 
static pANTLR3_STRING toStringSS (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
 
static pANTLR3_STRING toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop)
 

Detailed Description

Default implementation of CommonTokenStream.

Function Documentation

◆ _LA()

◆ antlr3CommonTokenDebugStreamSourceNew()

◆ antlr3CommonTokenStreamNew()

ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM antlr3CommonTokenStreamNew ( ANTLR3_UINT32  hint)

References _LA(), ANTLR3_INT_STREAM_struct::_LA, ANTLR3_TOKEN_STREAM_struct::_LT, ANTLR3_MALLOC, ANTLR3_TOKENSTREAM, antlr3IntStreamNew(), antlr3TokenStreamNew(), antlr3VectorNew(), consume(), ANTLR3_INT_STREAM_struct::consume, discardOffChannel(), ANTLR3_COMMON_TOKEN_STREAM_struct::discardOffChannelToks, discardTokenType(), ANTLR3_COMMON_TOKEN_STREAM_struct::discardTokenType, get(), ANTLR3_TOKEN_STREAM_struct::get, getSourceName(), ANTLR3_INT_STREAM_struct::getSourceName, getTokenRange(), ANTLR3_COMMON_TOKEN_STREAM_struct::getTokenRange, getTokens(), ANTLR3_COMMON_TOKEN_STREAM_struct::getTokens, getTokensList(), ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensList, getTokenSource(), ANTLR3_TOKEN_STREAM_struct::getTokenSource, getTokensSet(), ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensSet, getTokensType(), ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensType, ANTLR3_INT_STREAM_struct::index, ANTLR3_TOKEN_STREAM_struct::istream, mark(), ANTLR3_INT_STREAM_struct::mark, ANTLR3_COMMON_TOKEN_STREAM_struct::p, release(), ANTLR3_INT_STREAM_struct::release, reset(), ANTLR3_COMMON_TOKEN_STREAM_struct::reset, ANTLR3_INT_STREAM_struct::rewind, rewindLast(), ANTLR3_INT_STREAM_struct::rewindLast, rewindStream(), seek(), ANTLR3_INT_STREAM_struct::seek, setDebugListener(), ANTLR3_TOKEN_STREAM_struct::setDebugListener, setTokenSource(), ANTLR3_TOKEN_STREAM_struct::setTokenSource, setTokenTypeChannel(), ANTLR3_COMMON_TOKEN_STREAM_struct::setTokenTypeChannel, size(), ANTLR3_INT_STREAM_struct::size, ANTLR3_INT_STREAM_struct::super, ANTLR3_TOKEN_STREAM_struct::super, tindex(), ANTLR3_COMMON_TOKEN_STREAM_struct::tokens, tokLT(), toString(), ANTLR3_TOKEN_STREAM_struct::toString, toStringSS(), ANTLR3_TOKEN_STREAM_struct::toStringSS, toStringTT(), ANTLR3_TOKEN_STREAM_struct::toStringTT, ANTLR3_COMMON_TOKEN_STREAM_struct::tstream, and ANTLR3_INT_STREAM_struct::type.

Referenced by antlr3CommonTokenStreamSourceNew().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ antlr3CommonTokenStreamSourceNew()

◆ antlr3CTSFree()

◆ antlr3TokenStreamFree()

static void antlr3TokenStreamFree ( pANTLR3_TOKEN_STREAM  stream)
static

References ANTLR3_FREE.

Referenced by antlr3TokenStreamNew().

Here is the caller graph for this function:

◆ antlr3TokenStreamNew()

ANTLR3_API pANTLR3_TOKEN_STREAM antlr3TokenStreamNew ( )

References ANTLR3_MALLOC, antlr3TokenStreamFree(), and ANTLR3_TOKEN_STREAM_struct::free.

Referenced by antlr3CommonTokenStreamNew().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ consume()

static void consume ( pANTLR3_INT_STREAM  is)
static

Move the input pointer to the next incoming token.

The stream must become active with LT(1) available. consume() simply moves the input pointer so that LT(1) points at the next input symbol. Consume at least one token.

Walk past any token not on the channel the parser is listening to.

References ANTLR3_VECTOR_struct::count, ANTLR3_COMMON_TOKEN_STREAM_struct::p, skipOffTokenChannels(), ANTLR3_INT_STREAM_struct::super, ANTLR3_TOKEN_STREAM_struct::super, and ANTLR3_COMMON_TOKEN_STREAM_struct::tokens.

Referenced by antlr3CommonTokenStreamNew(), and dbgConsume().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ consumeInitialHiddenTokens()

static void consumeInitialHiddenTokens ( pANTLR3_INT_STREAM  is)
static

Debug only method to flag consumption of initial off-channel tokens in the input stream.

References ANTLR3_FALSE, ANTLR3_DEBUG_EVENT_LISTENER_struct::consumeHiddenToken, ANTLR3_TOKEN_STREAM_struct::debugger, ANTLR3_TOKEN_STREAM_struct::get, ANTLR3_INT_STREAM_struct::index, ANTLR3_TOKEN_STREAM_struct::initialStreamState, and ANTLR3_INT_STREAM_struct::super.

Referenced by dbgConsume(), dbgLA(), and dbgTokLT().

Here is the caller graph for this function:

◆ dbgConsume()

◆ dbgLA()

static ANTLR3_UINT32 dbgLA ( pANTLR3_INT_STREAM  is,
ANTLR3_INT32  i 
)
static

◆ dbgMark()

static ANTLR3_MARKER dbgMark ( pANTLR3_INT_STREAM  is)
static

◆ dbgRewindLast()

◆ dbgRewindStream()

◆ dbgSeek()

static void dbgSeek ( pANTLR3_INT_STREAM  is,
ANTLR3_MARKER  index 
)
static

References seek().

Referenced by antlr3CommonTokenDebugStreamSourceNew(), and setDebugListener().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ dbgTokLT()

static pANTLR3_COMMON_TOKEN dbgTokLT ( pANTLR3_TOKEN_STREAM  ts,
ANTLR3_INT32  k 
)
static

As per the normal tokLT but sends information to the debugger.

References ANTLR3_TRUE, consumeInitialHiddenTokens(), ANTLR3_TOKEN_STREAM_struct::initialStreamState, ANTLR3_TOKEN_STREAM_struct::istream, and tokLT().

Referenced by antlr3CommonTokenDebugStreamSourceNew(), and setDebugListener().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ discardOffChannel()

static void discardOffChannel ( pANTLR3_COMMON_TOKEN_STREAM  cts,
ANTLR3_BOOLEAN  discard 
)
static

References ANTLR3_COMMON_TOKEN_STREAM_struct::discardOffChannel.

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

◆ discardTokenType()

static void discardTokenType ( pANTLR3_COMMON_TOKEN_STREAM  cts,
ANTLR3_INT32  ttype 
)
static

References ANTLR3_FUNC_PTR, antlr3ListNew(), ANTLR3_COMMON_TOKEN_STREAM_struct::discardSet, and ANTLR3_LIST_struct::put.

Referenced by antlr3CommonTokenStreamNew().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ fillBuffer()

◆ fillBufferExt()

ANTLR3_API void fillBufferExt ( pANTLR3_COMMON_TOKEN_STREAM  tokenStream)

References fillBuffer().

Here is the call graph for this function:

◆ get()

◆ getSourceName()

static pANTLR3_STRING getSourceName ( pANTLR3_INT_STREAM  is)
static

Return a string that represents the name assoicated with the input source.

/param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream.

/returns /implements ANTLR3_INT_STREAM_struct::getSourceName()

References ANTLR3_INT_STREAM_struct::super.

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

◆ getTokenRange()

static pANTLR3_LIST getTokenRange ( pANTLR3_COMMON_TOKEN_STREAM  cts,
ANTLR3_UINT32  start,
ANTLR3_UINT32  stop 
)
static

References ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensSet.

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

◆ getTokens()

static pANTLR3_VECTOR getTokens ( pANTLR3_COMMON_TOKEN_STREAM  cts)
static

References fillBuffer(), ANTLR3_COMMON_TOKEN_STREAM_struct::p, and ANTLR3_COMMON_TOKEN_STREAM_struct::tokens.

Referenced by antlr3CommonTokenStreamNew().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ getTokensList()

static pANTLR3_LIST getTokensList ( pANTLR3_COMMON_TOKEN_STREAM  cts,
ANTLR3_UINT32  start,
ANTLR3_UINT32  stop,
pANTLR3_LIST  list 
)
static

References antlr3BitsetList(), ANTLR3_BITSET_struct::free, ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensSet, and ANTLR3_LIST_struct::table.

Referenced by antlr3CommonTokenStreamNew().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ getTokenSource()

static pANTLR3_TOKEN_SOURCE getTokenSource ( pANTLR3_TOKEN_STREAM  ts)
static

References ANTLR3_TOKEN_STREAM_struct::tokenSource.

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

◆ getTokensSet()

static pANTLR3_LIST getTokensSet ( pANTLR3_COMMON_TOKEN_STREAM  tokenStream,
ANTLR3_UINT32  start,
ANTLR3_UINT32  stop,
pANTLR3_BITSET  types 
)
static

Given a start and stop index, return a List of all tokens in the token type BitSet.

Return null if no tokens were found. This method looks at both on and off channel tokens.

References ANTLR3_TRUE, antlr3ListNew(), fillBuffer(), ANTLR3_LIST_struct::free, ANTLR3_TOKEN_STREAM_struct::get, ANTLR3_COMMON_TOKEN_struct::getType, ANTLR3_BITSET_struct::isMember, ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_COMMON_TOKEN_STREAM_struct::p, ANTLR3_LIST_struct::put, ANTLR3_INT_STREAM_struct::size, ANTLR3_LIST_struct::size, and ANTLR3_COMMON_TOKEN_STREAM_struct::tstream.

Referenced by antlr3CommonTokenStreamNew().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ getTokensType()

static pANTLR3_LIST getTokensType ( pANTLR3_COMMON_TOKEN_STREAM  cts,
ANTLR3_UINT32  start,
ANTLR3_UINT32  stop,
ANTLR3_UINT32  type 
)
static

References antlr3BitsetOf(), ANTLR3_BITSET_struct::free, and ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensSet.

Referenced by antlr3CommonTokenStreamNew().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ LB()

static pANTLR3_COMMON_TOKEN LB ( pANTLR3_COMMON_TOKEN_STREAM  tokenStream,
ANTLR3_INT32  i 
)
static

References ANTLR3_VECTOR_ELEMENT_struct::element, ANTLR3_VECTOR_struct::elements, fillBuffer(), ANTLR3_COMMON_TOKEN_STREAM_struct::p, skipOffTokenChannelsReverse(), and ANTLR3_COMMON_TOKEN_STREAM_struct::tokens.

Referenced by tokLT().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ mark()

static ANTLR3_MARKER mark ( pANTLR3_INT_STREAM  is)
static

References ANTLR3_INT_STREAM_struct::index, and ANTLR3_INT_STREAM_struct::lastMarker.

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

◆ release()

static void release ( pANTLR3_INT_STREAM  is,
ANTLR3_MARKER  mark 
)
static

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

◆ reset()

◆ rewindLast()

static void rewindLast ( pANTLR3_INT_STREAM  is)
static

References ANTLR3_INT_STREAM_struct::lastMarker, and ANTLR3_INT_STREAM_struct::rewind.

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

◆ rewindStream()

static void rewindStream ( pANTLR3_INT_STREAM  is,
ANTLR3_MARKER  marker 
)
static

References ANTLR3_INT_STREAM_struct::seek.

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

◆ seek()

static void seek ( pANTLR3_INT_STREAM  is,
ANTLR3_MARKER  index 
)
static

◆ setDebugListener()

◆ setTokenSource()

static void setTokenSource ( pANTLR3_TOKEN_STREAM  ts,
pANTLR3_TOKEN_SOURCE  tokenSource 
)
static

References ANTLR3_TOKEN_STREAM_struct::tokenSource.

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function:

◆ setTokenTypeChannel()

static void setTokenTypeChannel ( pANTLR3_COMMON_TOKEN_STREAM  tokenStream,
ANTLR3_UINT32  ttype,
ANTLR3_UINT32  channel 
)
static

A simple filter mechanism whereby you can tell this token stream to force all tokens of type ttype to be on channel.

For example, when interpreting, we cannot execute actions so we need to tell the stream to force all WS and NEWLINE to be a different, ignored, channel.

References ANTLR3_FUNC_PTR, antlr3ListNew(), ANTLR3_COMMON_TOKEN_STREAM_struct::channelOverrides, and ANTLR3_LIST_struct::put.

Referenced by antlr3CommonTokenStreamNew().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ size()

◆ skipOffTokenChannels()

◆ skipOffTokenChannelsReverse()

◆ tindex()

static ANTLR3_MARKER tindex ( pANTLR3_INT_STREAM  is)
static

◆ tokLT()

◆ toString()

◆ toStringSS()

◆ toStringTT()

static pANTLR3_STRING toStringTT ( pANTLR3_TOKEN_STREAM  ts,
pANTLR3_COMMON_TOKEN  start,
pANTLR3_COMMON_TOKEN  stop 
)
static

References ANTLR3_COMMON_TOKEN_struct::getTokenIndex, and ANTLR3_TOKEN_STREAM_struct::toStringSS.

Referenced by antlr3CommonTokenStreamNew().

Here is the caller graph for this function: