class Tokenizer
extends java.lang.Object
Modifier and Type | Field and Description |
---|---|
private static java.lang.String[] |
ALTERNATIVE_OPERATOR_NAMES |
(package private) java.lang.String |
expressionString |
private static byte[] |
FLAGS |
private static byte |
IS_ALPHA |
private static byte |
IS_DIGIT |
private static byte |
IS_HEXDIGIT |
(package private) int |
max |
(package private) int |
pos |
(package private) java.util.List<Token> |
tokens |
(package private) char[] |
toProcess |
Constructor and Description |
---|
Tokenizer(java.lang.String inputdata) |
Modifier and Type | Method and Description |
---|---|
java.util.List<Token> |
getTokens() |
private boolean |
isAlphabetic(char ch) |
private boolean |
isChar(char a,
char b) |
private boolean |
isDigit(char ch) |
private boolean |
isDoubleSuffix(char ch) |
private boolean |
isExponentChar(char ch) |
private boolean |
isFloatSuffix(char ch) |
private boolean |
isHexadecimalDigit(char ch) |
private boolean |
isIdentifier(char ch) |
private boolean |
isSign(char ch) |
private boolean |
isTwoCharToken(TokenKind kind)
Check if this might be a two character token.
|
private void |
lexDoubleQuotedStringLiteral() |
private void |
lexIdentifier() |
private void |
lexNumericLiteral(boolean firstCharIsZero) |
private void |
lexQuotedStringLiteral() |
void |
process() |
private void |
pushCharToken(TokenKind kind)
Push a token of just one character in length.
|
private void |
pushHexIntToken(char[] data,
boolean isLong,
int start,
int end) |
private void |
pushIntToken(char[] data,
boolean isLong,
int start,
int end) |
private void |
pushOneCharOrTwoCharToken(TokenKind kind,
int pos,
char[] data) |
private void |
pushPairToken(TokenKind kind)
Push a token of two characters in length.
|
private void |
pushRealToken(char[] data,
boolean isFloat,
int start,
int end) |
private char[] |
subarray(int start,
int end) |
private static final java.lang.String[] ALTERNATIVE_OPERATOR_NAMES
private static final byte[] FLAGS
private static final byte IS_DIGIT
private static final byte IS_HEXDIGIT
private static final byte IS_ALPHA
java.lang.String expressionString
char[] toProcess
int pos
int max
java.util.List<Token> tokens
public void process()
public java.util.List<Token> getTokens()
private void lexQuotedStringLiteral()
private void lexDoubleQuotedStringLiteral()
private void lexNumericLiteral(boolean firstCharIsZero)
private void lexIdentifier()
private void pushIntToken(char[] data, boolean isLong, int start, int end)
private void pushHexIntToken(char[] data, boolean isLong, int start, int end)
private void pushRealToken(char[] data, boolean isFloat, int start, int end)
private char[] subarray(int start, int end)
private boolean isTwoCharToken(TokenKind kind)
private void pushCharToken(TokenKind kind)
private void pushPairToken(TokenKind kind)
private void pushOneCharOrTwoCharToken(TokenKind kind, int pos, char[] data)
private boolean isIdentifier(char ch)
private boolean isChar(char a, char b)
private boolean isExponentChar(char ch)
private boolean isFloatSuffix(char ch)
private boolean isDoubleSuffix(char ch)
private boolean isSign(char ch)
private boolean isDigit(char ch)
private boolean isAlphabetic(char ch)
private boolean isHexadecimalDigit(char ch)