diff --git a/csharp/ProtocolBuffers.Test/TextFormatTest.cs b/csharp/ProtocolBuffers.Test/TextFormatTest.cs index a8cd35572d..8d30b96ccf 100644 --- a/csharp/ProtocolBuffers.Test/TextFormatTest.cs +++ b/csharp/ProtocolBuffers.Test/TextFormatTest.cs @@ -218,6 +218,37 @@ namespace Google.ProtocolBuffers { TestUtil.AssertAllExtensionsSet(builder.Build()); } + [Test] + public void ParseCompatibility() { + string original = "repeated_float: inf\n" + + "repeated_float: -inf\n" + + "repeated_float: nan\n" + + "repeated_float: inff\n" + + "repeated_float: -inff\n" + + "repeated_float: nanf\n" + + "repeated_float: 1.0f\n" + + "repeated_float: infinityf\n" + + "repeated_float: -Infinityf\n" + + "repeated_double: infinity\n" + + "repeated_double: -infinity\n" + + "repeated_double: nan\n"; + string canonical = "repeated_float: Infinity\n" + + "repeated_float: -Infinity\n" + + "repeated_float: NaN\n" + + "repeated_float: Infinity\n" + + "repeated_float: -Infinity\n" + + "repeated_float: NaN\n" + + "repeated_float: 1\n" + // Java has 1.0; this is fine + "repeated_float: Infinity\n" + + "repeated_float: -Infinity\n" + + "repeated_double: Infinity\n" + + "repeated_double: -Infinity\n" + + "repeated_double: NaN\n"; + TestAllTypes.Builder builder = TestAllTypes.CreateBuilder(); + TextFormat.Merge(original, builder); + Assert.AreEqual(canonical, builder.Build().ToString()); + } + [Test] public void ParseExotic() { TestAllTypes.Builder builder = TestAllTypes.CreateBuilder(); @@ -259,6 +290,19 @@ namespace Google.ProtocolBuffers { Assert.AreEqual(1, builder.OptionalGroup.A); } + [Test] + public void ParseComment() { + TestAllTypes.Builder builder = TestAllTypes.CreateBuilder(); + TextFormat.Merge( + "# this is a comment\n" + + "optional_int32: 1 # another comment\n" + + "optional_int64: 2\n" + + "# EOF comment", builder); + Assert.AreEqual(1, builder.OptionalInt32); + Assert.AreEqual(2, builder.OptionalInt64); + } + + private static void AssertParseError(string error, string text) { TestAllTypes.Builder builder = TestAllTypes.CreateBuilder(); try { diff --git a/csharp/ProtocolBuffers/TextFormat.cs b/csharp/ProtocolBuffers/TextFormat.cs index 36c8c02241..14f1ec73f0 100644 --- a/csharp/ProtocolBuffers/TextFormat.cs +++ b/csharp/ProtocolBuffers/TextFormat.cs @@ -40,8 +40,6 @@ namespace Google.ProtocolBuffers { /// /// Outputs a textual representation of to . /// - /// - /// public static void Print(UnknownFieldSet fields, TextWriter output) { TextGenerator generator = new TextGenerator(output); PrintUnknownFields(fields, generator); @@ -564,7 +562,7 @@ namespace Google.ProtocolBuffers { break; case FieldType.Float: - value = tokenizer.consumeFloat(); + value = tokenizer.ConsumeFloat(); break; case FieldType.Double: diff --git a/csharp/ProtocolBuffers/TextTokenizer.cs b/csharp/ProtocolBuffers/TextTokenizer.cs index 81c83c7d52..d53ae596a5 100644 --- a/csharp/ProtocolBuffers/TextTokenizer.cs +++ b/csharp/ProtocolBuffers/TextTokenizer.cs @@ -53,13 +53,18 @@ namespace Google.ProtocolBuffers { /// private int previousColumn = 0; - private static Regex WhitespaceAndCommentPattern = new Regex("\\G(\\s|(#[^\\\n]*\\n))+", RegexOptions.Compiled); - private static Regex TokenPattern = new Regex( + private static readonly Regex WhitespaceAndCommentPattern = new Regex("\\G(\\s|(#.*$))+", + RegexOptions.Compiled | RegexOptions.Multiline); + private static readonly Regex TokenPattern = new Regex( "\\G[a-zA-Z_][0-9a-zA-Z_+-]*|" + // an identifier "\\G[0-9+-][0-9a-zA-Z_.+-]*|" + // a number - "\\G\"([^\"\\\n\\\\]|\\\\[^\\\n])*(\"|\\\\?$)|" + // a double-quoted string - "\\G\'([^\"\\\n\\\\]|\\\\[^\\\n])*(\'|\\\\?$)", // a single-quoted string - RegexOptions.Compiled); + "\\G\"([^\"\\\n\\\\]|\\\\.)*(\"|\\\\?$)|" + // a double-quoted string + "\\G\'([^\"\\\n\\\\]|\\\\.)*(\'|\\\\?$)", // a single-quoted string + RegexOptions.Compiled | RegexOptions.Multiline); + + private static readonly Regex DoubleInfinity = new Regex("^-?inf(inity)?$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex FloatInfinity = new Regex("^-?inf(inity)?f?$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex FloatNan = new Regex("^nanf?$", RegexOptions.Compiled | RegexOptions.IgnoreCase); /** Construct a tokenizer that parses tokens from the given text. */ public TextTokenizer(string text) { @@ -243,6 +248,18 @@ namespace Google.ProtocolBuffers { /// Otherwise, throw a FormatException. /// public double ConsumeDouble() { + // We need to parse infinity and nan separately because + // double.Parse() does not accept "inf", "infinity", or "nan". + if (DoubleInfinity.IsMatch(currentToken)) { + bool negative = currentToken.StartsWith("-"); + NextToken(); + return negative ? double.NegativeInfinity : double.PositiveInfinity; + } + if (currentToken.Equals("nan", StringComparison.InvariantCultureIgnoreCase)) { + NextToken(); + return Double.NaN; + } + try { double result = double.Parse(currentToken, CultureInfo.InvariantCulture); NextToken(); @@ -258,7 +275,24 @@ namespace Google.ProtocolBuffers { /// If the next token is a float, consume it and return its value. /// Otherwise, throw a FormatException. /// - public float consumeFloat() { + public float ConsumeFloat() { + + // We need to parse infinity and nan separately because + // Float.parseFloat() does not accept "inf", "infinity", or "nan". + if (FloatInfinity.IsMatch(currentToken)) { + bool negative = currentToken.StartsWith("-"); + NextToken(); + return negative ? float.NegativeInfinity : float.PositiveInfinity; + } + if (FloatNan.IsMatch(currentToken)) { + NextToken(); + return float.NaN; + } + + if (currentToken.EndsWith("f")) { + currentToken = currentToken.TrimEnd('f'); + } + try { float result = float.Parse(currentToken, CultureInfo.InvariantCulture); NextToken();