ANTLR3C
3.3.1
|
Default implementation of CommonTokenStream. More...
#include <antlr3tokenstream.h>
Default implementation of CommonTokenStream.
|
static |
References ANTLR3_TOKEN_STREAM_struct::_LT, ANTLR3_TOKEN_INVALID, ANTLR3_COMMON_TOKEN_struct::getType, and ANTLR3_INT_STREAM_struct::super.
Referenced by antlr3CommonTokenStreamNew(), and dbgLA().
ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM antlr3CommonTokenDebugStreamSourceNew | ( | ANTLR3_UINT32 | hint, |
pANTLR3_TOKEN_SOURCE | source, | ||
pANTLR3_DEBUG_EVENT_LISTENER | debugger | ||
) |
References ANTLR3_INT_STREAM_struct::_LA, ANTLR3_TOKEN_STREAM_struct::_LT, ANTLR3_API, ANTLR3_FALSE, antlr3CommonTokenStreamSourceNew(), ANTLR3_INT_STREAM_struct::consume, dbgConsume(), dbgLA(), dbgMark(), dbgRewindLast(), dbgRewindStream(), dbgSeek(), dbgTokLT(), ANTLR3_TOKEN_STREAM_struct::debugger, ANTLR3_TOKEN_STREAM_struct::initialStreamState, ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_INT_STREAM_struct::mark, ANTLR3_INT_STREAM_struct::rewind, ANTLR3_INT_STREAM_struct::rewindLast, ANTLR3_INT_STREAM_struct::seek, and ANTLR3_COMMON_TOKEN_STREAM_struct::tstream.
ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM antlr3CommonTokenStreamNew | ( | ANTLR3_UINT32 | hint | ) |
References _LA(), ANTLR3_INT_STREAM_struct::_LA, ANTLR3_TOKEN_STREAM_struct::_LT, ANTLR3_MALLOC, ANTLR3_TOKENSTREAM, antlr3IntStreamNew(), antlr3TokenStreamNew(), antlr3VectorNew(), consume(), ANTLR3_INT_STREAM_struct::consume, discardOffChannel(), ANTLR3_COMMON_TOKEN_STREAM_struct::discardOffChannelToks, discardTokenType(), ANTLR3_COMMON_TOKEN_STREAM_struct::discardTokenType, ANTLR3_TOKEN_STREAM_struct::get, getSourceName(), ANTLR3_INT_STREAM_struct::getSourceName, getTokenRange(), ANTLR3_COMMON_TOKEN_STREAM_struct::getTokenRange, getTokens(), ANTLR3_COMMON_TOKEN_STREAM_struct::getTokens, getTokensList(), ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensList, getTokenSource(), ANTLR3_TOKEN_STREAM_struct::getTokenSource, getTokensSet(), ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensSet, getTokensType(), ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensType, ANTLR3_INT_STREAM_struct::index, ANTLR3_TOKEN_STREAM_struct::istream, mark(), ANTLR3_INT_STREAM_struct::mark, ANTLR3_COMMON_TOKEN_STREAM_struct::p, release(), ANTLR3_INT_STREAM_struct::release, reset(), ANTLR3_COMMON_TOKEN_STREAM_struct::reset, ANTLR3_INT_STREAM_struct::rewind, rewindLast(), ANTLR3_INT_STREAM_struct::rewindLast, rewindStream(), seek(), ANTLR3_INT_STREAM_struct::seek, setDebugListener(), ANTLR3_TOKEN_STREAM_struct::setDebugListener, setTokenSource(), ANTLR3_TOKEN_STREAM_struct::setTokenSource, setTokenTypeChannel(), ANTLR3_COMMON_TOKEN_STREAM_struct::setTokenTypeChannel, size(), ANTLR3_INT_STREAM_struct::size, ANTLR3_INT_STREAM_struct::super, ANTLR3_TOKEN_STREAM_struct::super, tindex(), ANTLR3_COMMON_TOKEN_STREAM_struct::tokens, tokLT(), toString(), ANTLR3_TOKEN_STREAM_struct::toString, toStringSS(), ANTLR3_TOKEN_STREAM_struct::toStringSS, toStringTT(), ANTLR3_TOKEN_STREAM_struct::toStringTT, ANTLR3_COMMON_TOKEN_STREAM_struct::tstream, and ANTLR3_INT_STREAM_struct::type.
Referenced by antlr3CommonTokenStreamSourceNew().
ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM antlr3CommonTokenStreamSourceNew | ( | ANTLR3_UINT32 | hint, |
pANTLR3_TOKEN_SOURCE | source | ||
) |
References ANTLR3_API, ANTLR3_FALSE, ANTLR3_TOKEN_DEFAULT_CHANNEL, antlr3CommonTokenStreamNew(), antlr3CTSFree(), ANTLR3_COMMON_TOKEN_STREAM_struct::channel, ANTLR3_COMMON_TOKEN_STREAM_struct::channelOverrides, ANTLR3_COMMON_TOKEN_STREAM_struct::discardOffChannel, ANTLR3_COMMON_TOKEN_STREAM_struct::discardSet, ANTLR3_COMMON_TOKEN_STREAM_struct::free, ANTLR3_TOKEN_STREAM_struct::setTokenSource, and ANTLR3_COMMON_TOKEN_STREAM_struct::tstream.
Referenced by antlr3CommonTokenDebugStreamSourceNew().
|
static |
References ANTLR3_FREE, ANTLR3_COMMON_TOKEN_STREAM_struct::channelOverrides, ANTLR3_COMMON_TOKEN_STREAM_struct::discardSet, ANTLR3_TOKEN_STREAM_struct::free, ANTLR3_LIST_struct::free, ANTLR3_INT_STREAM_struct::free, ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_INT_STREAM_struct::super, ANTLR3_TOKEN_STREAM_struct::super, ANTLR3_COMMON_TOKEN_STREAM_struct::tokens, and ANTLR3_COMMON_TOKEN_STREAM_struct::tstream.
Referenced by antlr3CommonTokenStreamSourceNew().
|
static |
References ANTLR3_FREE.
Referenced by antlr3TokenStreamNew().
ANTLR3_API pANTLR3_TOKEN_STREAM antlr3TokenStreamNew | ( | ) |
References ANTLR3_MALLOC, antlr3TokenStreamFree(), and ANTLR3_TOKEN_STREAM_struct::free.
Referenced by antlr3CommonTokenStreamNew().
|
static |
Move the input pointer to the next incoming token.
The stream must become active with LT(1) available. consume() simply moves the input pointer so that LT(1) points at the next input symbol. Consume at least one token.
Walk past any token not on the channel the parser is listening to.
References ANTLR3_VECTOR_struct::count, ANTLR3_COMMON_TOKEN_STREAM_struct::p, skipOffTokenChannels(), ANTLR3_INT_STREAM_struct::super, ANTLR3_TOKEN_STREAM_struct::super, and ANTLR3_COMMON_TOKEN_STREAM_struct::tokens.
Referenced by antlr3CommonTokenStreamNew(), and dbgConsume().
|
static |
Debug only method to flag consumption of initial off-channel tokens in the input stream.
References ANTLR3_FALSE, ANTLR3_DEBUG_EVENT_LISTENER_struct::consumeHiddenToken, ANTLR3_TOKEN_STREAM_struct::debugger, ANTLR3_TOKEN_STREAM_struct::get, ANTLR3_INT_STREAM_struct::index, ANTLR3_TOKEN_STREAM_struct::initialStreamState, and ANTLR3_INT_STREAM_struct::super.
Referenced by dbgConsume(), dbgLA(), and dbgTokLT().
|
static |
As per ordinary consume but notifies the debugger about hidden tokens and so on.
References ANTLR3_TOKEN_STREAM_struct::_LT, ANTLR3_TRUE, consume(), ANTLR3_DEBUG_EVENT_LISTENER_struct::consumeHiddenToken, consumeInitialHiddenTokens(), ANTLR3_DEBUG_EVENT_LISTENER_struct::consumeToken, ANTLR3_TOKEN_STREAM_struct::debugger, ANTLR3_TOKEN_STREAM_struct::get, ANTLR3_INT_STREAM_struct::index, ANTLR3_TOKEN_STREAM_struct::initialStreamState, and ANTLR3_INT_STREAM_struct::super.
Referenced by antlr3CommonTokenDebugStreamSourceNew(), and setDebugListener().
|
static |
As per _LA() but for debug mode.
References _LA(), ANTLR3_TRUE, consumeInitialHiddenTokens(), ANTLR3_TOKEN_STREAM_struct::debugger, ANTLR3_TOKEN_STREAM_struct::initialStreamState, ANTLR3_DEBUG_EVENT_LISTENER_struct::LT, ANTLR3_INT_STREAM_struct::super, and tokLT().
Referenced by antlr3CommonTokenDebugStreamSourceNew(), and setDebugListener().
|
static |
As per mark() but with a call to tell the debugger we are doing this.
References ANTLR3_TOKEN_STREAM_struct::debugger, ANTLR3_INT_STREAM_struct::index, ANTLR3_INT_STREAM_struct::lastMarker, ANTLR3_DEBUG_EVENT_LISTENER_struct::mark, and ANTLR3_INT_STREAM_struct::super.
Referenced by antlr3CommonTokenDebugStreamSourceNew(), and setDebugListener().
|
static |
References ANTLR3_TOKEN_STREAM_struct::debugger, ANTLR3_INT_STREAM_struct::lastMarker, ANTLR3_INT_STREAM_struct::rewind, ANTLR3_DEBUG_EVENT_LISTENER_struct::rewindLast, and ANTLR3_INT_STREAM_struct::super.
Referenced by antlr3CommonTokenDebugStreamSourceNew(), and setDebugListener().
|
static |
References ANTLR3_TOKEN_STREAM_struct::debugger, ANTLR3_DEBUG_EVENT_LISTENER_struct::rewind, ANTLR3_INT_STREAM_struct::seek, and ANTLR3_INT_STREAM_struct::super.
Referenced by antlr3CommonTokenDebugStreamSourceNew(), and setDebugListener().
|
static |
References ANTLR3_API, and seek().
Referenced by antlr3CommonTokenDebugStreamSourceNew(), and setDebugListener().
|
static |
As per the normal tokLT but sends information to the debugger.
References ANTLR3_TRUE, consumeInitialHiddenTokens(), ANTLR3_TOKEN_STREAM_struct::initialStreamState, ANTLR3_TOKEN_STREAM_struct::istream, and tokLT().
Referenced by antlr3CommonTokenDebugStreamSourceNew(), and setDebugListener().
|
static |
References ANTLR3_COMMON_TOKEN_STREAM_struct::discardOffChannel.
Referenced by antlr3CommonTokenStreamNew().
|
static |
References ANTLR3_FUNC_PTR, antlr3ListNew(), ANTLR3_COMMON_TOKEN_STREAM_struct::discardSet, and ANTLR3_LIST_struct::put.
Referenced by antlr3CommonTokenStreamNew().
|
static |
References ANTLR3_VECTOR_struct::add, ANTLR3_FALSE, ANTLR3_TOKEN_EOF, ANTLR3_TRUE, ANTLR3_UINT32_CAST, ANTLR3_INT_STREAM_struct::cachedSize, ANTLR3_COMMON_TOKEN_STREAM_struct::channel, ANTLR3_COMMON_TOKEN_STREAM_struct::channelOverrides, ANTLR3_VECTOR_struct::count, ANTLR3_COMMON_TOKEN_STREAM_struct::discardOffChannel, ANTLR3_COMMON_TOKEN_STREAM_struct::discardSet, ANTLR3_LIST_struct::get, ANTLR3_COMMON_TOKEN_struct::getChannel, ANTLR3_COMMON_TOKEN_struct::getType, ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_TOKEN_SOURCE_struct::nextToken, ANTLR3_COMMON_TOKEN_STREAM_struct::p, ANTLR3_COMMON_TOKEN_struct::setChannel, ANTLR3_COMMON_TOKEN_struct::setTokenIndex, skipOffTokenChannels(), ANTLR3_COMMON_TOKEN_STREAM_struct::tokens, ANTLR3_TOKEN_STREAM_struct::tokenSource, ANTLR3_COMMON_TOKEN_STREAM_struct::tstream, and ANTLR3_COMMON_TOKEN_struct::type.
Referenced by fillBufferExt(), getTokens(), getTokensSet(), LB(), tokLT(), toString(), and toStringSS().
ANTLR3_API void fillBufferExt | ( | pANTLR3_COMMON_TOKEN_STREAM | tokenStream | ) |
|
static |
|
static |
Return a string that represents the name assoicated with the input source.
/param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream.
/returns /implements ANTLR3_INT_STREAM_struct::getSourceName()
References ANTLR3_INT_STREAM_struct::super.
Referenced by antlr3CommonTokenStreamNew().
|
static |
References ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensSet.
Referenced by antlr3CommonTokenStreamNew().
|
static |
References fillBuffer(), ANTLR3_COMMON_TOKEN_STREAM_struct::p, and ANTLR3_COMMON_TOKEN_STREAM_struct::tokens.
Referenced by antlr3CommonTokenStreamNew().
|
static |
References antlr3BitsetList(), ANTLR3_BITSET_struct::free, ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensSet, and ANTLR3_LIST_struct::table.
Referenced by antlr3CommonTokenStreamNew().
|
static |
References ANTLR3_TOKEN_STREAM_struct::tokenSource.
Referenced by antlr3CommonTokenStreamNew().
|
static |
Given a start and stop index, return a List of all tokens in the token type BitSet.
Return null if no tokens were found. This method looks at both on and off channel tokens.
References ANTLR3_TRUE, antlr3ListNew(), fillBuffer(), ANTLR3_LIST_struct::free, ANTLR3_TOKEN_STREAM_struct::get, ANTLR3_COMMON_TOKEN_struct::getType, ANTLR3_BITSET_struct::isMember, ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_COMMON_TOKEN_STREAM_struct::p, ANTLR3_LIST_struct::put, ANTLR3_INT_STREAM_struct::size, ANTLR3_LIST_struct::size, and ANTLR3_COMMON_TOKEN_STREAM_struct::tstream.
Referenced by antlr3CommonTokenStreamNew().
|
static |
References antlr3BitsetOf(), ANTLR3_BITSET_struct::free, and ANTLR3_COMMON_TOKEN_STREAM_struct::getTokensSet.
Referenced by antlr3CommonTokenStreamNew().
|
static |
References ANTLR3_VECTOR_ELEMENT_struct::element, ANTLR3_VECTOR_struct::elements, fillBuffer(), ANTLR3_COMMON_TOKEN_STREAM_struct::p, skipOffTokenChannelsReverse(), and ANTLR3_COMMON_TOKEN_STREAM_struct::tokens.
Referenced by tokLT().
|
static |
References ANTLR3_INT_STREAM_struct::index, and ANTLR3_INT_STREAM_struct::lastMarker.
Referenced by antlr3CommonTokenStreamNew().
|
static |
|
static |
References ANTLR3_API, ANTLR3_FALSE, ANTLR3_TOKEN_DEFAULT_CHANNEL, antlr3VectorNew(), ANTLR3_COMMON_TOKEN_STREAM_struct::channel, ANTLR3_COMMON_TOKEN_STREAM_struct::channelOverrides, ANTLR3_VECTOR_struct::clear, ANTLR3_COMMON_TOKEN_STREAM_struct::discardOffChannel, ANTLR3_COMMON_TOKEN_STREAM_struct::discardSet, ANTLR3_LIST_struct::free, ANTLR3_COMMON_TOKEN_STREAM_struct::p, and ANTLR3_COMMON_TOKEN_STREAM_struct::tokens.
Referenced by antlr3CommonTokenStreamNew().
|
static |
References ANTLR3_INT_STREAM_struct::lastMarker, and ANTLR3_INT_STREAM_struct::rewind.
Referenced by antlr3CommonTokenStreamNew().
|
static |
References ANTLR3_INT_STREAM_struct::seek.
Referenced by antlr3CommonTokenStreamNew().
|
static |
References ANTLR3_COMMON_TOKEN_STREAM_struct::p, ANTLR3_INT_STREAM_struct::super, and ANTLR3_TOKEN_STREAM_struct::super.
Referenced by antlr3CommonTokenStreamNew(), and dbgSeek().
|
static |
References ANTLR3_INT_STREAM_struct::_LA, ANTLR3_TOKEN_STREAM_struct::_LT, ANTLR3_FALSE, ANTLR3_INT_STREAM_struct::consume, dbgConsume(), dbgLA(), dbgMark(), dbgRewindLast(), dbgRewindStream(), dbgSeek(), dbgTokLT(), ANTLR3_TOKEN_STREAM_struct::debugger, ANTLR3_TOKEN_STREAM_struct::initialStreamState, ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_INT_STREAM_struct::mark, ANTLR3_INT_STREAM_struct::rewind, ANTLR3_INT_STREAM_struct::rewindLast, and ANTLR3_INT_STREAM_struct::seek.
Referenced by antlr3CommonTokenStreamNew().
|
static |
References ANTLR3_TOKEN_STREAM_struct::tokenSource.
Referenced by antlr3CommonTokenStreamNew().
|
static |
A simple filter mechanism whereby you can tell this token stream to force all tokens of type ttype to be on channel.
For example, when interpreting, we cannot execute actions so we need to tell the stream to force all WS and NEWLINE to be a different, ignored, channel.
References ANTLR3_FUNC_PTR, antlr3ListNew(), ANTLR3_COMMON_TOKEN_STREAM_struct::channelOverrides, and ANTLR3_LIST_struct::put.
Referenced by antlr3CommonTokenStreamNew().
|
static |
References ANTLR3_INT_STREAM_struct::cachedSize, ANTLR3_VECTOR_struct::count, ANTLR3_INT_STREAM_struct::super, ANTLR3_TOKEN_STREAM_struct::super, and ANTLR3_COMMON_TOKEN_STREAM_struct::tokens.
Referenced by antlr3CommonTokenStreamNew().
|
static |
Given a starting index, return the index of the first on-channel token.
References ANTLR3_INT_STREAM_struct::cachedSize, ANTLR3_COMMON_TOKEN_struct::channel, ANTLR3_COMMON_TOKEN_STREAM_struct::channel, ANTLR3_VECTOR_ELEMENT_struct::element, ANTLR3_VECTOR_struct::elements, ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_COMMON_TOKEN_STREAM_struct::tokens, and ANTLR3_COMMON_TOKEN_STREAM_struct::tstream.
Referenced by consume(), fillBuffer(), and tokLT().
|
static |
References ANTLR3_COMMON_TOKEN_struct::channel, ANTLR3_COMMON_TOKEN_STREAM_struct::channel, ANTLR3_VECTOR_ELEMENT_struct::element, ANTLR3_VECTOR_struct::elements, and ANTLR3_COMMON_TOKEN_STREAM_struct::tokens.
Referenced by LB().
|
static |
References ANTLR3_COMMON_TOKEN_STREAM_struct::p, ANTLR3_INT_STREAM_struct::super, and ANTLR3_TOKEN_STREAM_struct::super.
Referenced by antlr3CommonTokenStreamNew().
|
static |
Get the ith token from the current position 1..n where k=1 is the first symbol of lookahead.
References ANTLR3_INT_STREAM_struct::cachedSize, ANTLR3_VECTOR_ELEMENT_struct::element, ANTLR3_VECTOR_struct::elements, ANTLR3_TOKEN_SOURCE_struct::eofToken, fillBuffer(), ANTLR3_INT_STREAM_struct::index, ANTLR3_TOKEN_STREAM_struct::istream, LB(), ANTLR3_COMMON_TOKEN_STREAM_struct::p, ANTLR3_COMMON_TOKEN_struct::setStartIndex, ANTLR3_COMMON_TOKEN_struct::setStopIndex, skipOffTokenChannels(), ANTLR3_TOKEN_STREAM_struct::super, ANTLR3_COMMON_TOKEN_STREAM_struct::tokens, and ANTLR3_TOKEN_STREAM_struct::tokenSource.
Referenced by antlr3CommonTokenStreamNew(), dbgLA(), and dbgTokLT().
|
static |
References fillBuffer(), ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_COMMON_TOKEN_STREAM_struct::p, ANTLR3_INT_STREAM_struct::size, ANTLR3_TOKEN_STREAM_struct::super, and ANTLR3_TOKEN_STREAM_struct::toStringSS.
Referenced by antlr3CommonTokenStreamNew().
|
static |
References fillBuffer(), ANTLR3_TOKEN_STREAM_struct::get, ANTLR3_COMMON_TOKEN_struct::getText, ANTLR3_TOKEN_STREAM_struct::getTokenSource, ANTLR3_TOKEN_STREAM_struct::istream, ANTLR3_STRING_FACTORY_struct::newRaw, ANTLR3_COMMON_TOKEN_STREAM_struct::p, ANTLR3_INT_STREAM_struct::size, ANTLR3_TOKEN_SOURCE_struct::strFactory, ANTLR3_TOKEN_STREAM_struct::super, and ANTLR3_COMMON_TOKEN_STREAM_struct::tokens.
Referenced by antlr3CommonTokenStreamNew().
|
static |
References ANTLR3_COMMON_TOKEN_struct::getTokenIndex, and ANTLR3_TOKEN_STREAM_struct::toStringSS.
Referenced by antlr3CommonTokenStreamNew().