public abstract class AbstractTokenizer extends Object
Modifier and Type | Field and Description |
---|---|
protected String |
expressionString
The string to be tokenized.
|
protected int[] |
linebreaks
Positions of linebreaks in the parsed string.
|
protected int |
max
Length of input data.
|
protected int |
pos
Current lexing position in the input data.
|
protected List<Token> |
tokens
Output stream of tokens.
|
protected char[] |
toProcess
The expressionString as a char array.
|
Constructor and Description |
---|
AbstractTokenizer() |
Modifier and Type | Method and Description |
---|---|
protected void |
addLinebreak() |
Tokens |
getTokens(String inputData) |
protected boolean |
isAlphabetic(char ch) |
protected boolean |
isArgValueIdentifierTerminator(char ch,
boolean quoteOpen)
For the variant tokenizer (used following an '=' to parse an argument value) we
only terminate that identifier if encountering a small set of characters.
|
protected boolean |
isDigit(char ch) |
protected boolean |
isIdentifier(char ch) |
protected boolean |
isQuote(char ch) |
protected boolean |
isTwoCharToken(TokenKind kind)
Check if this might be a two character token.
|
protected boolean |
isWhitespace(char ch) |
protected void |
lexArgValueIdentifier()
To prevent the need to quote all argument values, this identifier lexing function
is used just after an '=' when we are about to digest an arg value.
|
protected void |
lexDoubleQuotedStringLiteral()
Lex a string literal which uses double quotes as delimiters.
|
protected void |
lexIdentifier() |
protected void |
lexQuotedStringLiteral()
Lex a string literal which uses single quotes as delimiters.
|
protected void |
pushCharToken(TokenKind kind)
Push a token of just one character in length.
|
protected void |
pushPairToken(TokenKind kind)
Push a token of two characters in length.
|
void |
raiseException(DSLMessage message,
Object... inserts) |
protected boolean |
sameQuotes(int pos1,
int pos2) |
protected char[] |
subArray(int start,
int end) |
String |
toString() |
protected String expressionString
protected char[] toProcess
protected int max
protected int pos
protected int[] linebreaks
protected boolean isTwoCharToken(TokenKind kind)
protected void pushCharToken(TokenKind kind)
protected void pushPairToken(TokenKind kind)
protected boolean isIdentifier(char ch)
protected boolean isQuote(char ch)
protected boolean isWhitespace(char ch)
protected boolean isDigit(char ch)
protected boolean isAlphabetic(char ch)
protected char[] subArray(int start, int end)
protected void lexIdentifier()
protected void lexQuotedStringLiteral()
protected void lexDoubleQuotedStringLiteral()
protected boolean isArgValueIdentifierTerminator(char ch, boolean quoteOpen)
protected void lexArgValueIdentifier()
protected boolean sameQuotes(int pos1, int pos2)
public void raiseException(DSLMessage message, Object... inserts)
protected void addLinebreak()
Copyright © 2017 Pivotal Software, Inc.. All rights reserved.