From afe844bc95f8a0b2399315e618b636a6f32191dd Mon Sep 17 00:00:00 2001 From: csharptest Date: Fri, 10 Jun 2011 16:03:22 -0500 Subject: [PATCH] Added the JsonFormatWriter/Reader --- src/ProtoBench/Program.cs | 22 +- .../CompatTests/JsonCompatibilityTests.cs | 26 ++ .../ProtocolBuffers.Test.csproj | 2 + .../ProtocolBuffers.Test.csproj.user | 5 + .../TestWriterFormatJson.cs | 342 ++++++++++++++++++ src/ProtocolBuffers/ProtocolBuffers.csproj | 3 + .../Serialization/JsonFormatReader.cs | 181 +++++++++ .../Serialization/JsonFormatWriter.cs | 329 +++++++++++++++++ .../Serialization/JsonTextCursor.cs | 270 ++++++++++++++ 9 files changed, 1178 insertions(+), 2 deletions(-) create mode 100644 src/ProtocolBuffers.Test/CompatTests/JsonCompatibilityTests.cs create mode 100644 src/ProtocolBuffers.Test/ProtocolBuffers.Test.csproj.user create mode 100644 src/ProtocolBuffers.Test/TestWriterFormatJson.cs create mode 100644 src/ProtocolBuffers/Serialization/JsonFormatReader.cs create mode 100644 src/ProtocolBuffers/Serialization/JsonFormatWriter.cs create mode 100644 src/ProtocolBuffers/Serialization/JsonTextCursor.cs diff --git a/src/ProtoBench/Program.cs b/src/ProtoBench/Program.cs index 9de071f5e5..2239fe08b5 100644 --- a/src/ProtoBench/Program.cs +++ b/src/ProtoBench/Program.cs @@ -39,6 +39,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Threading; +using Google.ProtocolBuffers.Serialization; using Google.ProtocolBuffers.TestProtos; namespace Google.ProtocolBuffers.ProtoBench @@ -127,12 +128,25 @@ namespace Google.ProtocolBuffers.ProtoBench inputData = inputData ?? File.ReadAllBytes(file); MemoryStream inputStream = new MemoryStream(inputData); ByteString inputString = ByteString.CopyFrom(inputData); - IMessage sampleMessage = - defaultMessage.WeakCreateBuilderForType().WeakMergeFrom(inputString, registry).WeakBuild(); + IMessage sampleMessage = defaultMessage.WeakCreateBuilderForType().WeakMergeFrom(inputString, registry).WeakBuild(); + + StringWriter temp = new StringWriter(); + new XmlFormatWriter(temp).WriteMessage(sampleMessage); + string xmlMessageText = temp.ToString(); + temp = new StringWriter(); + new JsonFormatWriter(temp).WriteMessage(sampleMessage); + string jsonMessageText = temp.ToString(); + + //Serializers if(!FastTest) RunBenchmark("Serialize to byte string", inputData.Length, () => sampleMessage.ToByteString()); RunBenchmark("Serialize to byte array", inputData.Length, () => sampleMessage.ToByteArray()); if (!FastTest) RunBenchmark("Serialize to memory stream", inputData.Length, () => sampleMessage.WriteTo(new MemoryStream())); + + RunBenchmark("Serialize to xml", xmlMessageText.Length, () => new XmlFormatWriter(new StringWriter()).WriteMessage(sampleMessage)); + RunBenchmark("Serialize to json", jsonMessageText.Length, () => new JsonFormatWriter(new StringWriter()).WriteMessage(sampleMessage)); + + //Deserializers if (!FastTest) RunBenchmark("Deserialize from byte string", inputData.Length, () => defaultMessage.WeakCreateBuilderForType() .WeakMergeFrom(inputString, registry) @@ -151,6 +165,10 @@ namespace Google.ProtocolBuffers.ProtoBench CodedInputStream.CreateInstance(inputStream), registry) .WeakBuild(); }); + + RunBenchmark("Deserialize from xml", xmlMessageText.Length, () => new XmlFormatReader(xmlMessageText).Merge(defaultMessage.WeakCreateBuilderForType()).WeakBuild()); + RunBenchmark("Deserialize from json", jsonMessageText.Length, () => new JsonFormatReader(jsonMessageText).Merge(defaultMessage.WeakCreateBuilderForType()).WeakBuild()); + Console.WriteLine(); return true; } diff --git a/src/ProtocolBuffers.Test/CompatTests/JsonCompatibilityTests.cs b/src/ProtocolBuffers.Test/CompatTests/JsonCompatibilityTests.cs new file mode 100644 index 0000000000..30a08398a7 --- /dev/null +++ b/src/ProtocolBuffers.Test/CompatTests/JsonCompatibilityTests.cs @@ -0,0 +1,26 @@ +using System.IO; +using System.Text; +using Google.ProtocolBuffers.Serialization; +using NUnit.Framework; + +namespace Google.ProtocolBuffers.CompatTests +{ + [TestFixture] + public class JsonCompatibilityTests : CompatibilityTests + { + protected override object SerializeMessage(TMessage message) + { + StringWriter sw = new StringWriter(); + new JsonFormatWriter(sw) + .Formatted() + .WriteMessage(message); + return sw.ToString(); + } + + protected override TBuilder DeerializeMessage(object message, TBuilder builder, ExtensionRegistry registry) + { + new JsonFormatReader((string)message).Merge(builder); + return builder; + } + } +} \ No newline at end of file diff --git a/src/ProtocolBuffers.Test/ProtocolBuffers.Test.csproj b/src/ProtocolBuffers.Test/ProtocolBuffers.Test.csproj index 39b4026c86..199e410ad9 100644 --- a/src/ProtocolBuffers.Test/ProtocolBuffers.Test.csproj +++ b/src/ProtocolBuffers.Test/ProtocolBuffers.Test.csproj @@ -77,6 +77,7 @@ + True True @@ -115,6 +116,7 @@ + diff --git a/src/ProtocolBuffers.Test/ProtocolBuffers.Test.csproj.user b/src/ProtocolBuffers.Test/ProtocolBuffers.Test.csproj.user new file mode 100644 index 0000000000..b875c0c268 --- /dev/null +++ b/src/ProtocolBuffers.Test/ProtocolBuffers.Test.csproj.user @@ -0,0 +1,5 @@ + + + ProjectFiles + + \ No newline at end of file diff --git a/src/ProtocolBuffers.Test/TestWriterFormatJson.cs b/src/ProtocolBuffers.Test/TestWriterFormatJson.cs new file mode 100644 index 0000000000..1c95d8419b --- /dev/null +++ b/src/ProtocolBuffers.Test/TestWriterFormatJson.cs @@ -0,0 +1,342 @@ +using System; +using System.IO; +using Google.ProtocolBuffers.Serialization; +using NUnit.Framework; +using Google.ProtocolBuffers.TestProtos; + +namespace Google.ProtocolBuffers +{ + [TestFixture] + public class TestWriterFormatJson + { + protected string Content; + [System.Diagnostics.DebuggerNonUserCode] + protected void FormatterAssert(TMessage message, params string[] expecting) where TMessage : IMessageLite + { + StringWriter sw = new StringWriter(); + new JsonFormatWriter(sw).WriteMessage(message); + + Content = sw.ToString(); + + ExtensionRegistry registry = ExtensionRegistry.CreateInstance(); + UnitTestXmlSerializerTestProtoFile.RegisterAllExtensions(registry); + + IMessageLite copy = + new JsonFormatReader(Content) + .Merge(message.WeakCreateBuilderForType(), registry).WeakBuild(); + + Assert.AreEqual(typeof(TMessage), copy.GetType()); + Assert.AreEqual(message, copy); + foreach (string expect in expecting) + Assert.IsTrue(Content.IndexOf(expect) >= 0, "Expected to find content '{0}' in: \r\n{1}", expect, Content); + } + + [Test] + public void TestJsonFormatted() + { + TestXmlMessage message = TestXmlMessage.CreateBuilder() + .SetValid(true) + .SetNumber(0x1010) + .AddChildren(TestXmlMessage.Types.Children.CreateBuilder()) + .AddChildren(TestXmlMessage.Types.Children.CreateBuilder().AddOptions(EnumOptions.ONE)) + .AddChildren(TestXmlMessage.Types.Children.CreateBuilder().AddOptions(EnumOptions.ONE).AddOptions(EnumOptions.TWO)) + .AddChildren(TestXmlMessage.Types.Children.CreateBuilder().SetBinary(ByteString.CopyFromUtf8("abc"))) + .Build(); + + StringWriter sw = new StringWriter(); + new JsonFormatWriter(sw).Formatted() + .WriteMessage(message); + + string json = sw.ToString(); + + TestXmlMessage copy = new JsonFormatReader(json) + .Merge(TestXmlMessage.CreateBuilder()).Build(); + Assert.AreEqual(message, copy); + } + + [Test] + public void TestEmptyMessage() + { + FormatterAssert( + TestXmlChild.CreateBuilder() + .Build(), + @"{}" + ); + } + [Test] + public void TestRepeatedField() + { + FormatterAssert( + TestXmlChild.CreateBuilder() + .AddOptions(EnumOptions.ONE) + .AddOptions(EnumOptions.TWO) + .Build(), + @"{""options"":[""ONE"",""TWO""]}" + ); + } + [Test] + public void TestNestedEmptyMessage() + { + FormatterAssert( + TestXmlMessage.CreateBuilder() + .SetChild(TestXmlChild.CreateBuilder().Build()) + .Build(), + @"{""child"":{}}" + ); + } + [Test] + public void TestNestedMessage() + { + FormatterAssert( + TestXmlMessage.CreateBuilder() + .SetChild(TestXmlChild.CreateBuilder().AddOptions(EnumOptions.TWO).Build()) + .Build(), + @"{""child"":{""options"":[""TWO""]}}" + ); + } + [Test] + public void TestBooleanTypes() + { + FormatterAssert( + TestXmlMessage.CreateBuilder() + .SetValid(true) + .Build(), + @"{""valid"":true}" + ); + } + [Test] + public void TestFullMessage() + { + FormatterAssert( + TestXmlMessage.CreateBuilder() + .SetValid(true) + .SetText("text") + .AddTextlines("a") + .AddTextlines("b") + .AddTextlines("c") + .SetNumber(0x1010101010) + .AddNumbers(1) + .AddNumbers(2) + .AddNumbers(3) + .SetChild(TestXmlChild.CreateBuilder().AddOptions(EnumOptions.ONE).SetBinary(ByteString.CopyFrom(new byte[1]))) + .AddChildren(TestXmlMessage.Types.Children.CreateBuilder().AddOptions(EnumOptions.TWO).SetBinary(ByteString.CopyFrom(new byte[2]))) + .AddChildren(TestXmlMessage.Types.Children.CreateBuilder().AddOptions(EnumOptions.THREE).SetBinary(ByteString.CopyFrom(new byte[3]))) + .Build(), + @"""text"":""text""", + @"[""a"",""b"",""c""]", + @"[1,2,3]", + @"""child"":{", + @"""children"":[{", + @"AA==", + @"AAA=", + @"AAAA", + 0x1010101010L.ToString() + ); + } + [Test] + public void TestMessageWithXmlText() + { + FormatterAssert( + TestXmlMessage.CreateBuilder() + .SetText("") + .Build(), + @"{""text"":""<\/text>""}" + ); + } + [Test] + public void TestWithEscapeChars() + { + FormatterAssert( + TestXmlMessage.CreateBuilder() + .SetText(" \t <- \"leading space and trailing\" -> \\ \xef54 \x0000 \xFF \xFFFF \b \f \r \n \t ") + .Build(), + "{\"text\":\" \\t <- \\\"leading space and trailing\\\" -> \\\\ \\uef54 \\u0000 \\u00ff \\uffff \\b \\f \\r \\n \\t \"}" + ); + } + [Test] + public void TestWithExtensionText() + { + FormatterAssert( + TestXmlMessage.CreateBuilder() + .SetValid(false) + .SetExtension(UnitTestXmlSerializerTestProtoFile.ExtensionText, " extension text value ! ") + .Build(), + @"{""valid"":false,""extension_text"":"" extension text value ! ""}" + ); + } + [Test] + public void TestWithExtensionNumber() + { + FormatterAssert( + TestXmlMessage.CreateBuilder() + .SetExtension(UnitTestXmlSerializerTestProtoFile.ExtensionMessage, + new TestXmlExtension.Builder().SetNumber(42).Build()) + .Build(), + @"{""number"":42}" + ); + } + [Test] + public void TestWithExtensionArray() + { + FormatterAssert( + TestXmlMessage.CreateBuilder() + .AddExtension(UnitTestXmlSerializerTestProtoFile.ExtensionNumber, 100) + .AddExtension(UnitTestXmlSerializerTestProtoFile.ExtensionNumber, 101) + .AddExtension(UnitTestXmlSerializerTestProtoFile.ExtensionNumber, 102) + .Build(), + @"{""extension_number"":[100,101,102]}" + ); + } + [Test] + public void TestWithExtensionEnum() + { + FormatterAssert( + TestXmlMessage.CreateBuilder() + .SetExtension(UnitTestXmlSerializerTestProtoFile.ExtensionEnum, EnumOptions.ONE) + .Build(), + @"{""extension_enum"":""ONE""}" + ); + } + [Test] + public void TestMessageWithExtensions() + { + FormatterAssert( + TestXmlMessage.CreateBuilder() + .SetValid(true) + .SetText("text") + .SetExtension(UnitTestXmlSerializerTestProtoFile.ExtensionText, "extension text") + .SetExtension(UnitTestXmlSerializerTestProtoFile.ExtensionMessage, new TestXmlExtension.Builder().SetNumber(42).Build()) + .AddExtension(UnitTestXmlSerializerTestProtoFile.ExtensionNumber, 100) + .AddExtension(UnitTestXmlSerializerTestProtoFile.ExtensionNumber, 101) + .AddExtension(UnitTestXmlSerializerTestProtoFile.ExtensionNumber, 102) + .SetExtension(UnitTestXmlSerializerTestProtoFile.ExtensionEnum, EnumOptions.ONE) + .Build(), + @"""text"":""text""", + @"""valid"":true", + @"""extension_enum"":""ONE""", + @"""extension_text"":""extension text""", + @"""extension_number"":[100,101,102]", + @"""extension_message"":{""number"":42}" + ); + } + [Test] + public void TestMessageMissingExtensions() + { + TestXmlMessage original = TestXmlMessage.CreateBuilder() + .SetValid(true) + .SetText("text") + .SetExtension(UnitTestXmlSerializerTestProtoFile.ExtensionText, " extension text value ! ") + .SetExtension(UnitTestXmlSerializerTestProtoFile.ExtensionMessage, new TestXmlExtension.Builder().SetNumber(42).Build()) + .AddExtension(UnitTestXmlSerializerTestProtoFile.ExtensionNumber, 100) + .AddExtension(UnitTestXmlSerializerTestProtoFile.ExtensionNumber, 101) + .AddExtension(UnitTestXmlSerializerTestProtoFile.ExtensionNumber, 102) + .SetExtension(UnitTestXmlSerializerTestProtoFile.ExtensionEnum, EnumOptions.ONE) + .Build(); + + TestXmlMessage message = original.ToBuilder() + .ClearExtension(UnitTestXmlSerializerTestProtoFile.ExtensionText) + .ClearExtension(UnitTestXmlSerializerTestProtoFile.ExtensionMessage) + .ClearExtension(UnitTestXmlSerializerTestProtoFile.ExtensionNumber) + .ClearExtension(UnitTestXmlSerializerTestProtoFile.ExtensionEnum) + .Build(); + + JsonFormatWriter writer = new JsonFormatWriter(); + writer.WriteMessage(original); + Content = writer.ToString(); + + IMessageLite copy = new JsonFormatReader(Content) + .Merge(message.CreateBuilderForType()).Build(); + + Assert.AreNotEqual(original, message); + Assert.AreNotEqual(original, copy); + Assert.AreEqual(message, copy); + } + [Test] + public void TestMergeFields() + { + TestXmlMessage.Builder builder = TestXmlMessage.CreateBuilder(); + builder.MergeFrom(new JsonFormatReader("\"valid\": true")); + builder.MergeFrom(new JsonFormatReader("\"text\": \"text\", \"number\": \"411\"")); + Assert.AreEqual(true, builder.Valid); + Assert.AreEqual("text", builder.Text); + Assert.AreEqual(411, builder.Number); + } + [Test] + public void TestMessageArray() + { + JsonFormatWriter writer = new JsonFormatWriter().Formatted(); + using (writer.StartArray()) + { + writer.WriteMessage(TestXmlMessage.CreateBuilder().SetNumber(1).AddTextlines("a").Build()); + writer.WriteMessage(TestXmlMessage.CreateBuilder().SetNumber(2).AddTextlines("b").Build()); + writer.WriteMessage(TestXmlMessage.CreateBuilder().SetNumber(3).AddTextlines("c").Build()); + } + string json = writer.ToString(); + JsonFormatReader reader = new JsonFormatReader(json); + + TestXmlMessage.Builder builder = TestXmlMessage.CreateBuilder(); + int ordinal = 0; + + foreach (JsonFormatReader r in reader.EnumerateArray()) + { + r.Merge(builder); + Assert.AreEqual(++ordinal, builder.Number); + } + Assert.AreEqual(3, ordinal); + Assert.AreEqual(3, builder.TextlinesCount); + } + [Test] + public void TestNestedMessageArray() + { + JsonFormatWriter writer = new JsonFormatWriter(); + using (writer.StartArray()) + { + using (writer.StartArray()) + { + writer.WriteMessage(TestXmlMessage.CreateBuilder().SetNumber(1).AddTextlines("a").Build()); + writer.WriteMessage(TestXmlMessage.CreateBuilder().SetNumber(2).AddTextlines("b").Build()); + } + using (writer.StartArray()) + writer.WriteMessage(TestXmlMessage.CreateBuilder().SetNumber(3).AddTextlines("c").Build()); + } + string json = writer.ToString(); + JsonFormatReader reader = new JsonFormatReader(json); + + TestXmlMessage.Builder builder = TestXmlMessage.CreateBuilder(); + int ordinal = 0; + + foreach (JsonFormatReader r in reader.EnumerateArray()) + foreach (JsonFormatReader r2 in r.EnumerateArray()) + { + r2.Merge(builder); + Assert.AreEqual(++ordinal, builder.Number); + } + Assert.AreEqual(3, ordinal); + Assert.AreEqual(3, builder.TextlinesCount); + } + [Test, ExpectedException(typeof(FormatException))] + public void FailWithEmptyText() + { + new JsonFormatReader("") + .Merge(TestXmlMessage.CreateBuilder()); + } + [Test, ExpectedException(typeof(FormatException))] + public void FailWithUnexpectedValue() + { + new JsonFormatReader("{{}}") + .Merge(TestXmlMessage.CreateBuilder()); + } + [Test, ExpectedException(typeof(FormatException))] + public void FailWithUnQuotedName() + { + new JsonFormatReader("{name:{}}") + .Merge(TestXmlMessage.CreateBuilder()); + } + [Test, ExpectedException(typeof(FormatException))] + public void FailWithUnexpectedType() + { + new JsonFormatReader("{\"valid\":{}}") + .Merge(TestXmlMessage.CreateBuilder()); + } + } +} diff --git a/src/ProtocolBuffers/ProtocolBuffers.csproj b/src/ProtocolBuffers/ProtocolBuffers.csproj index 7321942041..19a97a46e1 100644 --- a/src/ProtocolBuffers/ProtocolBuffers.csproj +++ b/src/ProtocolBuffers/ProtocolBuffers.csproj @@ -184,6 +184,9 @@ + + + diff --git a/src/ProtocolBuffers/Serialization/JsonFormatReader.cs b/src/ProtocolBuffers/Serialization/JsonFormatReader.cs new file mode 100644 index 0000000000..e9df913899 --- /dev/null +++ b/src/ProtocolBuffers/Serialization/JsonFormatReader.cs @@ -0,0 +1,181 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Xml; + +namespace Google.ProtocolBuffers.Serialization +{ + /// + /// JsonFormatReader is used to parse Json into a message or an array of messages + /// + public class JsonFormatReader : AbstractTextReader + { + private readonly JsonTextCursor _input; + private readonly Stack _stopChar; + + enum ReaderState { Start, BeginValue, EndValue, BeginObject, BeginArray } + string _current; + ReaderState _state; + + /// + /// Constructs a JsonFormatReader to parse Json into a message + /// + public JsonFormatReader(string jsonText) + { + _input = new JsonTextCursor(jsonText.ToCharArray()); + _stopChar = new Stack(); + _stopChar.Push(-1); + _state = ReaderState.Start; + } + /// + /// Constructs a JsonFormatReader to parse Json into a message + /// + public JsonFormatReader(TextReader input) + { + _input = new JsonTextCursor(input); + _stopChar = new Stack(); + _stopChar.Push(-1); + _state = ReaderState.Start; + } + + /// + /// Returns true if the reader is currently on an array element + /// + public bool IsArrayMessage { get { return _input.NextChar == '['; } } + + /// + /// Returns an enumerator that is used to cursor over an array of messages + /// + /// + /// This is generally used when receiving an array of messages rather than a single root message + /// + public IEnumerable EnumerateArray() + { + foreach (string ignored in ForeachArrayItem(_current)) + yield return this; + } + + /// + /// Merges the contents of stream into the provided message builder + /// + public override TBuilder Merge(TBuilder builder, ExtensionRegistry registry) + { + _input.Consume('{'); + _stopChar.Push('}'); + + _state = ReaderState.BeginObject; + builder.WeakMergeFrom(this, registry); + _input.Consume((char)_stopChar.Pop()); + _state = ReaderState.EndValue; + return builder; + } + + /// + /// Causes the reader to skip past this field + /// + protected override void Skip() + { + object temp; + _input.ReadVariant(out temp); + _state = ReaderState.EndValue; + } + + /// + /// Peeks at the next field in the input stream and returns what information is available. + /// + /// + /// This may be called multiple times without actually reading the field. Only after the field + /// is either read, or skipped, should PeekNext return a different value. + /// + protected override bool PeekNext(out string field) + { + field = _current; + if(_state == ReaderState.BeginValue) + return true; + + int next = _input.NextChar; + if (next == _stopChar.Peek()) + return false; + + _input.Assert(next != -1, "Unexpected end of file."); + + //not sure about this yet, it will allow {, "a":true } + if (_state == ReaderState.EndValue && !_input.TryConsume(',')) + return false; + + field = _current = _input.ReadString(); + _input.Consume(':'); + _state = ReaderState.BeginValue; + return true; + } + + /// + /// Returns true if it was able to read a String from the input + /// + protected override bool ReadAsText(ref string value, Type typeInfo) + { + object temp; + JsonTextCursor.JsType type = _input.ReadVariant(out temp); + _state = ReaderState.EndValue; + + _input.Assert(type != JsonTextCursor.JsType.Array && type != JsonTextCursor.JsType.Object, "Encountered {0} while expecting {1}", type, typeInfo); + if (type == JsonTextCursor.JsType.Null) + return false; + if (type == JsonTextCursor.JsType.True) value = "1"; + else if (type == JsonTextCursor.JsType.False) value = "0"; + else value = temp as string; + + //exponent representation of integer number: + if (value != null && type == JsonTextCursor.JsType.Number && + (typeInfo != typeof(double) && typeInfo != typeof(float)) && + value.IndexOf("e", StringComparison.OrdinalIgnoreCase) > 0) + { + value = XmlConvert.ToString((long)Math.Round(XmlConvert.ToDouble(value), 0)); + } + return value != null; + } + + /// + /// Returns true if it was able to read a ByteString from the input + /// + protected override bool Read(ref ByteString value) + { + string bytes = null; + if (Read(ref bytes)) + { + value = ByteString.FromBase64(bytes); + return true; + } + return false; + } + + /// + /// Cursors through the array elements and stops at the end of the array + /// + protected override IEnumerable ForeachArrayItem(string field) + { + _input.Consume('['); + _stopChar.Push(']'); + _state = ReaderState.BeginArray; + while (_input.NextChar != ']') + { + _current = field; + yield return field; + if(!_input.TryConsume(',')) + break; + } + _input.Consume((char)_stopChar.Pop()); + _state = ReaderState.EndValue; + } + + /// + /// Merges the input stream into the provided IBuilderLite + /// + protected override bool ReadMessage(IBuilderLite builder, ExtensionRegistry registry) + { + Merge(builder, registry); + return true; + } + + } +} diff --git a/src/ProtocolBuffers/Serialization/JsonFormatWriter.cs b/src/ProtocolBuffers/Serialization/JsonFormatWriter.cs new file mode 100644 index 0000000000..5d5144ba02 --- /dev/null +++ b/src/ProtocolBuffers/Serialization/JsonFormatWriter.cs @@ -0,0 +1,329 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Google.ProtocolBuffers.Descriptors; + +namespace Google.ProtocolBuffers.Serialization +{ + /// + /// JsonFormatWriter is a .NET 2.0 friendly json formatter for proto buffer messages. For .NET 3.5 + /// you may also use the XmlFormatWriter with an XmlWriter created by the + /// JsonReaderWriterFactory. + /// + public class JsonFormatWriter : AbstractTextWriter + { + private readonly char[] _buffer; + private readonly TextWriter _output; + private readonly List _counter; + private bool _isArray; + int _bufferPos; + /// + /// Constructs a JsonFormatWriter to output to a new instance of a StringWriter, use + /// the ToString() member to extract the final Json on completion. + /// + public JsonFormatWriter() : this(new StringWriter()) { } + /// + /// Constructs a JsonFormatWriter to output to the given text writer + /// + public JsonFormatWriter(TextWriter output) + { + _buffer = new char[4096]; + _bufferPos = 0; + _output = output; + _counter = new List(); + _counter.Add(0); + } + + + private void WriteToOutput(string format, params object[] args) + { WriteToOutput(String.Format(format, args)); } + + private void WriteToOutput(string text) + { WriteToOutput(text.ToCharArray(), 0, text.Length); } + + private void WriteToOutput(char[] chars, int offset, int len) + { + if (_bufferPos + len >= _buffer.Length) + Flush(); + if (len < _buffer.Length) + { + if (len <= 12) + { + int stop = offset + len; + for (int i = offset; i < stop; i++) + _buffer[_bufferPos++] = chars[i]; + } + else + { + Buffer.BlockCopy(chars, offset << 1, _buffer, _bufferPos << 1, len << 1); + _bufferPos += len; + } + } + else + _output.Write(chars, offset, len); + } + + private void WriteToOutput(char ch) + { + if (_bufferPos >= _buffer.Length) + Flush(); + _buffer[_bufferPos++] = ch; + } + + public override void Flush() + { + if (_bufferPos > 0) + { + _output.Write(_buffer, 0, _bufferPos); + _bufferPos = 0; + } + base.Flush(); + } + + /// + /// Returns the output of TextWriter.ToString() where TextWriter is the ctor argument. + /// + public override string ToString() + { Flush(); return _output.ToString(); } + + /// Sets the output formatting to use Environment.NewLine with 4-character indentions + public JsonFormatWriter Formatted() + { + NewLine = Environment.NewLine; + Indent = " "; + Whitespace = " "; + return this; + } + + /// Gets or sets the characters to use for the new-line, default = empty + public string NewLine { get; set; } + /// Gets or sets the text to use for indenting, default = empty + public string Indent { get; set; } + /// Gets or sets the whitespace to use to separate the text, default = empty + public string Whitespace { get; set; } + + private void Seperator() + { + if (_counter.Count == 0) + throw new InvalidOperationException("Missmatched open/close in Json writer."); + + int index = _counter.Count - 1; + if (_counter[index] > 0) + WriteToOutput(','); + + WriteLine(String.Empty); + _counter[index] = _counter[index] + 1; + } + + private void WriteLine(string content) + { + if (!String.IsNullOrEmpty(NewLine)) + { + WriteToOutput(NewLine); + for (int i = 1; i < _counter.Count; i++) + WriteToOutput(Indent); + } + else if(!String.IsNullOrEmpty(Whitespace)) + WriteToOutput(Whitespace); + + WriteToOutput(content); + } + + private void WriteName(string field) + { + Seperator(); + if (!String.IsNullOrEmpty(field)) + { + WriteToOutput('"'); + WriteToOutput(field); + WriteToOutput('"'); + WriteToOutput(':'); + if (!String.IsNullOrEmpty(Whitespace)) + WriteToOutput(Whitespace); + } + } + + private void EncodeText(string value) + { + char[] text = value.ToCharArray(); + int len = text.Length; + int pos = 0; + + while (pos < len) + { + int next = pos; + while (next < len && text[next] >= 32 && text[next] < 127 && text[next] != '\\' && text[next] != '/' && text[next] != '"') + next++; + WriteToOutput(text, pos, next - pos); + if (next < len) + { + switch (text[next]) + { + case '"': WriteToOutput(@"\"""); break; + case '\\': WriteToOutput(@"\\"); break; + //odd at best to escape '/', most Json implementations don't, but it is defined in the rfc-4627 + case '/': WriteToOutput(@"\/"); break; + case '\b': WriteToOutput(@"\b"); break; + case '\f': WriteToOutput(@"\f"); break; + case '\n': WriteToOutput(@"\n"); break; + case '\r': WriteToOutput(@"\r"); break; + case '\t': WriteToOutput(@"\t"); break; + default: WriteToOutput(@"\u{0:x4}", (int)text[next]); break; + } + next++; + } + pos = next; + } + } + + /// + /// Writes a String value + /// + protected override void WriteAsText(string field, string textValue, object typedValue) + { + WriteName(field); + if(typedValue is bool || typedValue is int || typedValue is uint || typedValue is long || typedValue is ulong || typedValue is double || typedValue is float) + WriteToOutput(textValue); + else + { + WriteToOutput('"'); + if (typedValue is string) + EncodeText(textValue); + else + WriteToOutput(textValue); + WriteToOutput('"'); + } + } + + /// + /// Writes a Double value + /// + protected override void Write(string field, double value) + { + if (double.IsNaN(value) || double.IsNegativeInfinity(value) || double.IsPositiveInfinity(value)) + throw new InvalidOperationException("This format does not support NaN, Infinity, or -Infinity"); + base.Write(field, value); + } + + /// + /// Writes a Single value + /// + protected override void Write(string field, float value) + { + if (float.IsNaN(value) || float.IsNegativeInfinity(value) || float.IsPositiveInfinity(value)) + throw new InvalidOperationException("This format does not support NaN, Infinity, or -Infinity"); + base.Write(field, value); + } + + // Treat enum as string + protected override void WriteEnum(string field, int number, string name) + { + Write(field, name); + } + + /// + /// Writes an array of field values + /// + protected override void WriteArray(FieldType type, string field, System.Collections.IEnumerable items) + { + System.Collections.IEnumerator enumerator = items.GetEnumerator(); + try { if (!enumerator.MoveNext()) return; } + finally { if (enumerator is IDisposable) ((IDisposable)enumerator).Dispose(); } + + WriteName(field); + WriteToOutput("["); + _counter.Add(0); + + base.WriteArray(type, String.Empty, items); + + _counter.RemoveAt(_counter.Count - 1); + WriteLine("]"); + } + + /// + /// Writes a message + /// + protected override void WriteMessageOrGroup(string field, IMessageLite message) + { + WriteName(field); + WriteMessage(message); + } + + /// + /// Writes the message to the the formatted stream. + /// + public override void WriteMessage(IMessageLite message) + { + if (_isArray) Seperator(); + WriteToOutput("{"); + _counter.Add(0); + message.WriteTo(this); + _counter.RemoveAt(_counter.Count - 1); + WriteLine("}"); + Flush(); + } + + /// + /// Writes a message + /// + [System.ComponentModel.Browsable(false)] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + public override void WriteMessage(string field, IMessageLite message) + { + WriteMessage(message); + } + + /// + /// Used in streaming arrays of objects to the writer + /// + /// + /// + /// using(writer.StartArray()) + /// foreach(IMessageLite m in messages) + /// writer.WriteMessage(m); + /// + /// + public sealed class JsonArray : IDisposable + { + JsonFormatWriter _writer; + internal JsonArray(JsonFormatWriter writer) + { + _writer = writer; + _writer.WriteToOutput("["); + _writer._counter.Add(0); + } + + /// + /// Causes the end of the array character to be written. + /// + void EndArray() + { + if (_writer != null) + { + _writer._counter.RemoveAt(_writer._counter.Count - 1); + _writer.WriteLine("]"); + _writer.Flush(); + } + _writer = null; + } + void IDisposable.Dispose() { EndArray(); } + } + + /// + /// Used to write an array of messages as the output rather than a single message. + /// + /// + /// + /// using(writer.StartArray()) + /// foreach(IMessageLite m in messages) + /// writer.WriteMessage(m); + /// + /// + public JsonArray StartArray() + { + if (_isArray) Seperator(); + _isArray = true; + return new JsonArray(this); + } + } +} \ No newline at end of file diff --git a/src/ProtocolBuffers/Serialization/JsonTextCursor.cs b/src/ProtocolBuffers/Serialization/JsonTextCursor.cs new file mode 100644 index 0000000000..a2a5b73d97 --- /dev/null +++ b/src/ProtocolBuffers/Serialization/JsonTextCursor.cs @@ -0,0 +1,270 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; + +namespace Google.ProtocolBuffers.Serialization +{ + /// + /// JSon Tokenizer used by JsonFormatReader + /// + class JsonTextCursor + { + public enum JsType { String, Number, Object, Array, True, False, Null } + + private readonly char[] _buffer; + private int _bufferPos; + private readonly TextReader _input; + private int _lineNo, _linePos; + + public JsonTextCursor(char[] input) + { + _input = null; + _buffer = input; + _bufferPos = 0; + _next = Peek(); + _lineNo = 1; + } + + public JsonTextCursor(TextReader input) + { + _input = input; + _next = Peek(); + _lineNo = 1; + } + + private int Peek() + { + if (_input != null) + return _input.Peek(); + else if (_bufferPos < _buffer.Length) + return _buffer[_bufferPos]; + else + return -1; + } + + private int Read() + { + if (_input != null) + return _input.Read(); + else if (_bufferPos < _buffer.Length) + return _buffer[_bufferPos++]; + else + return -1; + } + + int _next; + public Char NextChar { get { SkipWhitespace(); return (char)_next; } } + + #region Assert(...) + [System.Diagnostics.DebuggerNonUserCode] + private string CharDisplay(int ch) + { + return ch == -1 ? "EOF" : + (ch > 32 && ch < 127) ? String.Format("'{0}'", (char)ch) : + String.Format("'\\u{0:x4}'", ch); + } + [System.Diagnostics.DebuggerNonUserCode] + private void Assert(bool cond, char expected) + { + if (!cond) + { + throw new FormatException( + String.Format(CultureInfo.InvariantCulture, + "({0}:{1}) error: Unexpected token {2}, expected: {3}.", + _lineNo, _linePos, + CharDisplay(_next), + CharDisplay(expected) + )); + } + } + [System.Diagnostics.DebuggerNonUserCode] + public void Assert(bool cond, string message) + { + if (!cond) + { + throw new FormatException( + String.Format(CultureInfo.InvariantCulture, + "({0},{1}) error: {2}", _lineNo, _linePos, message)); + } + } + [System.Diagnostics.DebuggerNonUserCode] + public void Assert(bool cond, string format, params object[] args) + { + if (!cond) + { + if (args != null && args.Length > 0) + format = String.Format(format, args); + throw new FormatException( + String.Format(CultureInfo.InvariantCulture, + "({0},{1}) error: {2}", _lineNo, _linePos, format)); + } + } + #endregion + + private char ReadChar() + { + int ch = Read(); + Assert(ch != -1, "Unexpected end of file."); + if (ch == '\n') + { + _lineNo++; + _linePos = 0; + } + else if (ch != '\r') + { + _linePos++; + } + _next = Peek(); + return (char)ch; + } + + public void Consume(char ch) { Assert(TryConsume(ch), ch); } + public bool TryConsume(char ch) + { + SkipWhitespace(); + if (_next == ch) + { + ReadChar(); + return true; + } + return false; + } + + public void Consume(string sequence) + { + SkipWhitespace(); + + foreach (char ch in sequence) + Assert(ch == ReadChar(), "Expected token '{0}'.", sequence); + } + + public void SkipWhitespace() + { + int chnext = _next; + while (chnext != -1) + { + if (!Char.IsWhiteSpace((char)chnext)) + break; + ReadChar(); + chnext = _next; + } + } + + public string ReadString() + { + SkipWhitespace(); + Consume('"'); + StringBuilder sb = new StringBuilder(); + while (_next != '"') + { + if (_next == '\\') + { + Consume('\\');//skip the escape + char ch = ReadChar(); + switch (ch) + { + case 'b': sb.Append('\b'); break; + case 'f': sb.Append('\f'); break; + case 'n': sb.Append('\n'); break; + case 'r': sb.Append('\r'); break; + case 't': sb.Append('\t'); break; + case 'u': + { + string hex = new string(new char[] { ReadChar(), ReadChar(), ReadChar(), ReadChar() }); + int result; + Assert(int.TryParse(hex, NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out result), + "Expected a 4-character hex specifier."); + sb.Append((char)result); + break; + } + default: + sb.Append(ch); break; + } + } + else + { + Assert(_next != '\n' && _next != '\r' && _next != '\f' && _next != -1, '"'); + sb.Append(ReadChar()); + } + } + Consume('"'); + return sb.ToString(); + } + + public string ReadNumber() + { + SkipWhitespace(); + + StringBuilder sb = new StringBuilder(); + if (_next == '-') + sb.Append(ReadChar()); + Assert(_next >= '0' && _next <= '9', "Expected a numeric type."); + while ((_next >= '0' && _next <= '9') || _next == '.') + sb.Append(ReadChar()); + if (_next == 'e' || _next == 'E') + { + sb.Append(ReadChar()); + if (_next == '-' || _next == '+') + sb.Append(ReadChar()); + Assert(_next >= '0' && _next <= '9', "Expected a numeric type."); + while (_next >= '0' && _next <= '9') + sb.Append(ReadChar()); + } + return sb.ToString(); + } + + public JsType ReadVariant(out object value) + { + SkipWhitespace(); + switch (_next) + { + case 'n': Consume("null"); value = null; return JsType.Null; + case 't': Consume("true"); value = true; return JsType.True; + case 'f': Consume("false"); value = false; return JsType.False; + case '"': value = ReadString(); return JsType.String; + case '{': + { + Consume('{'); + while (NextChar != '}') + { + ReadString(); + Consume(':'); + object tmp; + ReadVariant(out tmp); + if (!TryConsume(',')) + break; + } + Consume('}'); + value = null; + return JsType.Object; + } + case '[': + { + Consume('['); + List values = new List(); + while (NextChar != ']') + { + object tmp; + ReadVariant(out tmp); + values.Add(tmp); + if (!TryConsume(',')) + break; + } + Consume(']'); + value = values.ToArray(); + return JsType.Array; + } + default: + if ((_next >= '0' && _next <= '9') || _next == '-') + { + value = ReadNumber(); + return JsType.Number; + } + Assert(false, "Expected a value."); + throw new FormatException(); + } + } + } +} \ No newline at end of file