|
|
|
|
@ -0,0 +1,341 @@ |
|
|
|
|
using System; |
|
|
|
|
using System.Globalization; |
|
|
|
|
using System.Text.RegularExpressions; |
|
|
|
|
|
|
|
|
|
namespace Google.ProtocolBuffers { |
|
|
|
|
/// <summary> |
|
|
|
|
/// Represents a stream of tokens parsed from a string. |
|
|
|
|
/// </summary> |
|
|
|
|
internal sealed class TextTokenizer { |
|
|
|
|
private readonly string text; |
|
|
|
|
private string currentToken; |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// The character index within the text to perform the next regex match at. |
|
|
|
|
/// </summary> |
|
|
|
|
private int matchPos = 0; |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// The character index within the text at which the current token begins. |
|
|
|
|
/// </summary> |
|
|
|
|
private int pos = 0; |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// The line number of the current token. |
|
|
|
|
/// </summary> |
|
|
|
|
private int line = 0; |
|
|
|
|
/// <summary> |
|
|
|
|
/// The column number of the current token. |
|
|
|
|
/// </summary> |
|
|
|
|
private int column = 0; |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// The line number of the previous token. |
|
|
|
|
/// </summary> |
|
|
|
|
private int previousLine = 0; |
|
|
|
|
/// <summary> |
|
|
|
|
/// The column number of the previous token. |
|
|
|
|
/// </summary> |
|
|
|
|
private int previousColumn = 0; |
|
|
|
|
|
|
|
|
|
private static Regex WhitespaceAndCommentPattern = new Regex("\\G(\\s|(#[^\\\n]*\\n))+", RegexOptions.Compiled); |
|
|
|
|
private static Regex TokenPattern = new Regex( |
|
|
|
|
"\\G[a-zA-Z_][0-9a-zA-Z_+-]*|" + // an identifier |
|
|
|
|
"\\G[0-9+-][0-9a-zA-Z_.+-]*|" + // a number |
|
|
|
|
"\\G\"([^\"\\\n\\\\]|\\\\[^\\\n])*(\"|\\\\?$)|" + // a double-quoted string |
|
|
|
|
"\\G\'([^\"\\\n\\\\]|\\\\[^\\\n])*(\'|\\\\?$)", // a single-quoted string |
|
|
|
|
RegexOptions.Compiled); |
|
|
|
|
|
|
|
|
|
/** Construct a tokenizer that parses tokens from the given text. */ |
|
|
|
|
public TextTokenizer(string text) { |
|
|
|
|
this.text = text; |
|
|
|
|
SkipWhitespace(); |
|
|
|
|
NextToken(); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// Are we at the end of the input? |
|
|
|
|
/// </summary> |
|
|
|
|
public bool AtEnd { |
|
|
|
|
get { return currentToken.Length == 0; } |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// Advances to the next token. |
|
|
|
|
/// </summary> |
|
|
|
|
public void NextToken() { |
|
|
|
|
previousLine = line; |
|
|
|
|
previousColumn = column; |
|
|
|
|
|
|
|
|
|
// Advance the line counter to the current position. |
|
|
|
|
while (pos < matchPos) { |
|
|
|
|
if (text[pos] == '\n') { |
|
|
|
|
++line; |
|
|
|
|
column = 0; |
|
|
|
|
} else { |
|
|
|
|
++column; |
|
|
|
|
} |
|
|
|
|
++pos; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// Match the next token. |
|
|
|
|
if (matchPos == text.Length) { |
|
|
|
|
// EOF |
|
|
|
|
currentToken = ""; |
|
|
|
|
} else { |
|
|
|
|
Match match = TokenPattern.Match(text, matchPos); |
|
|
|
|
if (match.Success) { |
|
|
|
|
currentToken = match.Value; |
|
|
|
|
matchPos += match.Length; |
|
|
|
|
} else { |
|
|
|
|
// Take one character. |
|
|
|
|
currentToken = text[matchPos].ToString(); |
|
|
|
|
matchPos++; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
SkipWhitespace(); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// Skip over any whitespace so that matchPos starts at the next token. |
|
|
|
|
/// </summary> |
|
|
|
|
private void SkipWhitespace() { |
|
|
|
|
Match match = WhitespaceAndCommentPattern.Match(text, matchPos); |
|
|
|
|
if (match.Success) { |
|
|
|
|
matchPos += match.Length; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// If the next token exactly matches the given token, consume it and return |
|
|
|
|
/// true. Otherwise, return false without doing anything. |
|
|
|
|
/// </summary> |
|
|
|
|
public bool TryConsume(string token) { |
|
|
|
|
if (currentToken == token) { |
|
|
|
|
NextToken(); |
|
|
|
|
return true; |
|
|
|
|
} |
|
|
|
|
return false; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/* |
|
|
|
|
* If the next token exactly matches {@code token}, consume it. Otherwise, |
|
|
|
|
* throw a {@link ParseException}. |
|
|
|
|
*/ |
|
|
|
|
/// <summary> |
|
|
|
|
/// If the next token exactly matches the specified one, consume it. |
|
|
|
|
/// Otherwise, throw a FormatException. |
|
|
|
|
/// </summary> |
|
|
|
|
/// <param name="token"></param> |
|
|
|
|
public void Consume(string token) { |
|
|
|
|
if (!TryConsume(token)) { |
|
|
|
|
throw CreateFormatException("Expected \"" + token + "\"."); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// Returns true if the next token is an integer, but does not consume it. |
|
|
|
|
/// </summary> |
|
|
|
|
public bool LookingAtInteger() { |
|
|
|
|
if (currentToken.Length == 0) { |
|
|
|
|
return false; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
char c = currentToken[0]; |
|
|
|
|
return ('0' <= c && c <= '9') || c == '-' || c == '+'; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// If the next token is an identifier, consume it and return its value. |
|
|
|
|
/// Otherwise, throw a FormatException. |
|
|
|
|
/// </summary> |
|
|
|
|
public string ConsumeIdentifier() { |
|
|
|
|
foreach (char c in currentToken) { |
|
|
|
|
if (('a' <= c && c <= 'z') || |
|
|
|
|
('A' <= c && c <= 'Z') || |
|
|
|
|
('0' <= c && c <= '9') || |
|
|
|
|
(c == '_') || (c == '.')) { |
|
|
|
|
// OK |
|
|
|
|
} else { |
|
|
|
|
throw CreateFormatException("Expected identifier."); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
string result = currentToken; |
|
|
|
|
NextToken(); |
|
|
|
|
return result; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// If the next token is a 32-bit signed integer, consume it and return its |
|
|
|
|
/// value. Otherwise, throw a FormatException. |
|
|
|
|
/// </summary> |
|
|
|
|
public int ConsumeInt32() { |
|
|
|
|
try { |
|
|
|
|
int result = TextFormat.ParseInt32(currentToken); |
|
|
|
|
NextToken(); |
|
|
|
|
return result; |
|
|
|
|
} catch (FormatException e) { |
|
|
|
|
throw CreateIntegerParseException(e); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// If the next token is a 32-bit unsigned integer, consume it and return its |
|
|
|
|
/// value. Otherwise, throw a FormatException. |
|
|
|
|
/// </summary> |
|
|
|
|
public uint ConsumeUInt32() { |
|
|
|
|
try { |
|
|
|
|
uint result = TextFormat.ParseUInt32(currentToken); |
|
|
|
|
NextToken(); |
|
|
|
|
return result; |
|
|
|
|
} catch (FormatException e) { |
|
|
|
|
throw CreateIntegerParseException(e); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// If the next token is a 64-bit signed integer, consume it and return its |
|
|
|
|
/// value. Otherwise, throw a FormatException. |
|
|
|
|
/// </summary> |
|
|
|
|
public long ConsumeInt64() { |
|
|
|
|
try { |
|
|
|
|
long result = TextFormat.ParseInt64(currentToken); |
|
|
|
|
NextToken(); |
|
|
|
|
return result; |
|
|
|
|
} catch (FormatException e) { |
|
|
|
|
throw CreateIntegerParseException(e); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// If the next token is a 64-bit unsigned integer, consume it and return its |
|
|
|
|
/// value. Otherwise, throw a FormatException. |
|
|
|
|
/// </summary> |
|
|
|
|
public ulong ConsumeUInt64() { |
|
|
|
|
try { |
|
|
|
|
ulong result = TextFormat.ParseUInt64(currentToken); |
|
|
|
|
NextToken(); |
|
|
|
|
return result; |
|
|
|
|
} catch (FormatException e) { |
|
|
|
|
throw CreateIntegerParseException(e); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// If the next token is a double, consume it and return its value. |
|
|
|
|
/// Otherwise, throw a FormatException. |
|
|
|
|
/// </summary> |
|
|
|
|
public double ConsumeDouble() { |
|
|
|
|
try { |
|
|
|
|
double result = double.Parse(currentToken, CultureInfo.InvariantCulture); |
|
|
|
|
NextToken(); |
|
|
|
|
return result; |
|
|
|
|
} catch (FormatException e) { |
|
|
|
|
throw CreateFloatParseException(e); |
|
|
|
|
} catch (OverflowException e) { |
|
|
|
|
throw CreateFloatParseException(e); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// If the next token is a float, consume it and return its value. |
|
|
|
|
/// Otherwise, throw a FormatException. |
|
|
|
|
/// </summary> |
|
|
|
|
public float consumeFloat() { |
|
|
|
|
try { |
|
|
|
|
float result = float.Parse(currentToken, CultureInfo.InvariantCulture); |
|
|
|
|
NextToken(); |
|
|
|
|
return result; |
|
|
|
|
} catch (FormatException e) { |
|
|
|
|
throw CreateFloatParseException(e); |
|
|
|
|
} catch (OverflowException e) { |
|
|
|
|
throw CreateFloatParseException(e); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// If the next token is a Boolean, consume it and return its value. |
|
|
|
|
/// Otherwise, throw a FormatException. |
|
|
|
|
/// </summary> |
|
|
|
|
public bool ConsumeBoolean() { |
|
|
|
|
if (currentToken == "true") { |
|
|
|
|
NextToken(); |
|
|
|
|
return true; |
|
|
|
|
} |
|
|
|
|
if (currentToken == "false") { |
|
|
|
|
NextToken(); |
|
|
|
|
return false; |
|
|
|
|
} |
|
|
|
|
throw CreateFormatException("Expected \"true\" or \"false\"."); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// If the next token is a string, consume it and return its (unescaped) value. |
|
|
|
|
/// Otherwise, throw a FormatException. |
|
|
|
|
/// </summary> |
|
|
|
|
public string ConsumeString() { |
|
|
|
|
return ConsumeByteString().ToStringUtf8(); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// If the next token is a string, consume it, unescape it as a |
|
|
|
|
/// ByteString and return it. Otherwise, throw a FormatException. |
|
|
|
|
/// </summary> |
|
|
|
|
public ByteString ConsumeByteString() { |
|
|
|
|
char quote = currentToken.Length > 0 ? currentToken[0] : '\0'; |
|
|
|
|
if (quote != '\"' && quote != '\'') { |
|
|
|
|
throw CreateFormatException("Expected string."); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if (currentToken.Length < 2 || |
|
|
|
|
currentToken[currentToken.Length-1] != quote) { |
|
|
|
|
throw CreateFormatException("String missing ending quote."); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
try { |
|
|
|
|
string escaped = currentToken.Substring(1, currentToken.Length - 2); |
|
|
|
|
ByteString result = TextFormat.UnescapeBytes(escaped); |
|
|
|
|
NextToken(); |
|
|
|
|
return result; |
|
|
|
|
} catch (FormatException e) { |
|
|
|
|
throw CreateFormatException(e.Message); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// Returns a format exception with the current line and column numbers |
|
|
|
|
/// in the description, suitable for throwing. |
|
|
|
|
/// </summary> |
|
|
|
|
public FormatException CreateFormatException(string description) { |
|
|
|
|
// Note: People generally prefer one-based line and column numbers. |
|
|
|
|
return new FormatException((line + 1) + ":" + (column + 1) + ": " + description); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// Returns a format exception with the line and column numbers of the |
|
|
|
|
/// previous token in the description, suitable for throwing. |
|
|
|
|
/// </summary> |
|
|
|
|
public FormatException CreateFormatExceptionPreviousToken(string description) { |
|
|
|
|
// Note: People generally prefer one-based line and column numbers. |
|
|
|
|
return new FormatException((previousLine + 1) + ":" + (previousColumn + 1) + ": " + description); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// Constructs an appropriate FormatException for the given existing exception |
|
|
|
|
/// when trying to parse an integer. |
|
|
|
|
/// </summary> |
|
|
|
|
private FormatException CreateIntegerParseException(FormatException e) { |
|
|
|
|
return CreateFormatException("Couldn't parse integer: " + e.Message); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// <summary> |
|
|
|
|
/// Constructs an appropriate FormatException for the given existing exception |
|
|
|
|
/// when trying to parse a float or double. |
|
|
|
|
/// </summary> |
|
|
|
|
private FormatException CreateFloatParseException(Exception e) { |
|
|
|
|
return CreateFormatException("Couldn't parse number: " + e.Message); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
Jon Skeet
17 years ago