diff --git a/Makefile.am b/Makefile.am index 91a029b464..090e5037b0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -85,15 +85,15 @@ csharp_EXTRA_DIST= \ csharp/src/AddressBook/ListPeople.cs \ csharp/src/AddressBook/Program.cs \ csharp/src/AddressBook/SampleUsage.cs \ + csharp/src/Google.Protobuf.Benchmarks/BenchmarkDatasetConfig.cs \ csharp/src/Google.Protobuf.Benchmarks/BenchmarkMessage1Proto3.cs \ csharp/src/Google.Protobuf.Benchmarks/Benchmarks.cs \ csharp/src/Google.Protobuf.Benchmarks/Google.Protobuf.Benchmarks.csproj \ + csharp/src/Google.Protobuf.Benchmarks/GoogleMessageBenchmark.cs \ csharp/src/Google.Protobuf.Benchmarks/ParseRawPrimitivesBenchmark.cs \ + csharp/src/Google.Protobuf.Benchmarks/ParseMessagesBenchmark.cs \ csharp/src/Google.Protobuf.Benchmarks/Program.cs \ - csharp/src/Google.Protobuf.Benchmarks/SerializationBenchmark.cs \ - csharp/src/Google.Protobuf.Benchmarks/SerializationConfig.cs \ csharp/src/Google.Protobuf.Benchmarks/wrapper_benchmark_messages.proto \ - csharp/src/Google.Protobuf.Benchmarks/WrapperBenchmark.cs \ csharp/src/Google.Protobuf.Benchmarks/WrapperBenchmarkMessages.cs \ csharp/src/Google.Protobuf.Conformance/Conformance.cs \ csharp/src/Google.Protobuf.Conformance/Google.Protobuf.Conformance.csproj \ diff --git a/csharp/src/Google.Protobuf.Benchmarks/SerializationConfig.cs b/csharp/src/Google.Protobuf.Benchmarks/BenchmarkDatasetConfig.cs similarity index 89% rename from csharp/src/Google.Protobuf.Benchmarks/SerializationConfig.cs rename to csharp/src/Google.Protobuf.Benchmarks/BenchmarkDatasetConfig.cs index 679f16cb9a..c0754190b6 100644 --- a/csharp/src/Google.Protobuf.Benchmarks/SerializationConfig.cs +++ b/csharp/src/Google.Protobuf.Benchmarks/BenchmarkDatasetConfig.cs @@ -43,20 +43,20 @@ namespace Google.Protobuf.Benchmarks /// /// The configuration for a single serialization test, loaded from a dataset. /// - public class SerializationConfig + public class BenchmarkDatasetConfig { private static readonly Dictionary parsersByMessageName = - typeof(SerializationBenchmark).Assembly.GetTypes() + typeof(GoogleMessageBenchmark).Assembly.GetTypes() .Where(t => typeof(IMessage).IsAssignableFrom(t)) .ToDictionary( t => ((MessageDescriptor) t.GetProperty("Descriptor", BindingFlags.Static | BindingFlags.Public).GetValue(null)).FullName, t => ((MessageParser) t.GetProperty("Parser", BindingFlags.Static | BindingFlags.Public).GetValue(null))); public MessageParser Parser { get; } - public IEnumerable Payloads { get; } + public List Payloads { get; } public string Name { get; } - public SerializationConfig(string resource) + public BenchmarkDatasetConfig(string resource, string shortName = null) { var data = LoadData(resource); var dataset = BenchmarkDataset.Parser.ParseFrom(data); @@ -66,13 +66,13 @@ namespace Google.Protobuf.Benchmarks throw new ArgumentException($"No parser for message {dataset.MessageName} in this assembly"); } Parser = parser; - Payloads = dataset.Payload; - Name = dataset.Name; + Payloads = new List(dataset.Payload.Select(p => p.ToByteArray())); + Name = shortName ?? dataset.Name; } private static byte[] LoadData(string resource) { - using (var stream = typeof(SerializationBenchmark).Assembly.GetManifestResourceStream($"Google.Protobuf.Benchmarks.{resource}")) + using (var stream = typeof(GoogleMessageBenchmark).Assembly.GetManifestResourceStream($"Google.Protobuf.Benchmarks.{resource}")) { if (stream == null) { diff --git a/csharp/src/Google.Protobuf.Benchmarks/SerializationBenchmark.cs b/csharp/src/Google.Protobuf.Benchmarks/GoogleMessageBenchmark.cs similarity index 74% rename from csharp/src/Google.Protobuf.Benchmarks/SerializationBenchmark.cs rename to csharp/src/Google.Protobuf.Benchmarks/GoogleMessageBenchmark.cs index d8c2ec11d7..132967e00a 100644 --- a/csharp/src/Google.Protobuf.Benchmarks/SerializationBenchmark.cs +++ b/csharp/src/Google.Protobuf.Benchmarks/GoogleMessageBenchmark.cs @@ -38,23 +38,27 @@ using System.Linq; namespace Google.Protobuf.Benchmarks { /// - /// Benchmark for serializing (to a MemoryStream) and deserializing (from a ByteString). + /// Benchmark for serializing and deserializing of standard datasets that are also + /// measured by benchmarks in other languages. /// Over time we may wish to test the various different approaches to serialization and deserialization separately. + /// See https://github.com/protocolbuffers/protobuf/blob/master/benchmarks/README.md + /// See https://github.com/protocolbuffers/protobuf/blob/master/docs/performance.md /// [MemoryDiagnoser] - public class SerializationBenchmark + public class GoogleMessageBenchmark { /// - /// All the configurations to be tested. Add more datasets to the array as they're available. + /// All the datasets to be tested. Add more datasets to the array as they're available. /// (When C# supports proto2, this will increase significantly.) /// - public static SerializationConfig[] Configurations => new[] + public static BenchmarkDatasetConfig[] DatasetConfigurations => new[] { - new SerializationConfig("dataset.google_message1_proto3.pb") + // short name is specified to make results table more readable + new BenchmarkDatasetConfig("dataset.google_message1_proto3.pb", "goog_msg1_proto3") }; - [ParamsSource(nameof(Configurations))] - public SerializationConfig Configuration { get; set; } + [ParamsSource(nameof(DatasetConfigurations))] + public BenchmarkDatasetConfig Dataset { get; set; } private MessageParser parser; /// @@ -67,8 +71,8 @@ namespace Google.Protobuf.Benchmarks [GlobalSetup] public void GlobalSetup() { - parser = Configuration.Parser; - subTests = Configuration.Payloads.Select(p => new SubTest(p, parser.ParseFrom(p))).ToList(); + parser = Dataset.Parser; + subTests = Dataset.Payloads.Select(p => new SubTest(p, parser.ParseFrom(p))).ToList(); } [Benchmark] @@ -78,7 +82,7 @@ namespace Google.Protobuf.Benchmarks public void ToByteArray() => subTests.ForEach(item => item.ToByteArray()); [Benchmark] - public void ParseFromByteString() => subTests.ForEach(item => item.ParseFromByteString(parser)); + public void ParseFromByteArray() => subTests.ForEach(item => item.ParseFromByteArray(parser)); [Benchmark] public void ParseFromStream() => subTests.ForEach(item => item.ParseFromStream(parser)); @@ -87,13 +91,13 @@ namespace Google.Protobuf.Benchmarks { private readonly Stream destinationStream; private readonly Stream sourceStream; - private readonly ByteString data; + private readonly byte[] data; private readonly IMessage message; - public SubTest(ByteString data, IMessage message) + public SubTest(byte[] data, IMessage message) { destinationStream = new MemoryStream(data.Length); - sourceStream = new MemoryStream(data.ToByteArray()); + sourceStream = new MemoryStream(data); this.data = data; this.message = message; } @@ -108,7 +112,7 @@ namespace Google.Protobuf.Benchmarks public void ToByteArray() => message.ToByteArray(); - public void ParseFromByteString(MessageParser parser) => parser.ParseFrom(data); + public void ParseFromByteArray(MessageParser parser) => parser.ParseFrom(data); public void ParseFromStream(MessageParser parser) { diff --git a/csharp/src/Google.Protobuf.Benchmarks/ParseMessagesBenchmark.cs b/csharp/src/Google.Protobuf.Benchmarks/ParseMessagesBenchmark.cs new file mode 100644 index 0000000000..cbc47328fa --- /dev/null +++ b/csharp/src/Google.Protobuf.Benchmarks/ParseMessagesBenchmark.cs @@ -0,0 +1,170 @@ +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2019 Google Inc. All rights reserved. +// https://github.com/protocolbuffers/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +using BenchmarkDotNet.Attributes; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Buffers; +using Google.Protobuf.WellKnownTypes; + +namespace Google.Protobuf.Benchmarks +{ + /// + /// Benchmark that tests parsing performance for various messages. + /// + [MemoryDiagnoser] + public class ParseMessagesBenchmark + { + const int MaxMessages = 100; + + SubTest manyWrapperFieldsTest = new SubTest(CreateManyWrapperFieldsMessage(), ManyWrapperFieldsMessage.Parser, () => new ManyWrapperFieldsMessage(), MaxMessages); + SubTest manyPrimitiveFieldsTest = new SubTest(CreateManyPrimitiveFieldsMessage(), ManyPrimitiveFieldsMessage.Parser, () => new ManyPrimitiveFieldsMessage(), MaxMessages); + SubTest emptyMessageTest = new SubTest(new Empty(), Empty.Parser, () => new Empty(), MaxMessages); + + public IEnumerable MessageCountValues => new[] { 10, 100 }; + + [GlobalSetup] + public void GlobalSetup() + { + } + + [Benchmark] + public IMessage ManyWrapperFieldsMessage_ParseFromByteArray() + { + return manyWrapperFieldsTest.ParseFromByteArray(); + } + + [Benchmark] + public IMessage ManyPrimitiveFieldsMessage_ParseFromByteArray() + { + return manyPrimitiveFieldsTest.ParseFromByteArray(); + } + + [Benchmark] + public IMessage EmptyMessage_ParseFromByteArray() + { + return emptyMessageTest.ParseFromByteArray(); + } + + [Benchmark] + [ArgumentsSource(nameof(MessageCountValues))] + public void ManyWrapperFieldsMessage_ParseDelimitedMessagesFromByteArray(int messageCount) + { + manyWrapperFieldsTest.ParseDelimitedMessagesFromByteArray(messageCount); + } + + [Benchmark] + [ArgumentsSource(nameof(MessageCountValues))] + public void ManyPrimitiveFieldsMessage_ParseDelimitedMessagesFromByteArray(int messageCount) + { + manyPrimitiveFieldsTest.ParseDelimitedMessagesFromByteArray(messageCount); + } + + private static ManyWrapperFieldsMessage CreateManyWrapperFieldsMessage() + { + // Example data match data of an internal benchmarks + return new ManyWrapperFieldsMessage() + { + Int64Field19 = 123, + Int64Field37 = 1000032, + Int64Field26 = 3453524500, + DoubleField79 = 1.2, + DoubleField25 = 234, + DoubleField9 = 123.3, + DoubleField28 = 23, + DoubleField7 = 234, + DoubleField50 = 2.45 + }; + } + + private static ManyPrimitiveFieldsMessage CreateManyPrimitiveFieldsMessage() + { + // Example data match data of an internal benchmarks + return new ManyPrimitiveFieldsMessage() + { + Int64Field19 = 123, + Int64Field37 = 1000032, + Int64Field26 = 3453524500, + DoubleField79 = 1.2, + DoubleField25 = 234, + DoubleField9 = 123.3, + DoubleField28 = 23, + DoubleField7 = 234, + DoubleField50 = 2.45 + }; + } + + private class SubTest + { + private readonly IMessage message; + private readonly MessageParser parser; + private readonly Func factory; + private readonly byte[] data; + private readonly byte[] multipleMessagesData; + + public SubTest(IMessage message, MessageParser parser, Func factory, int maxMessageCount) + { + this.message = message; + this.parser = parser; + this.factory = factory; + this.data = message.ToByteArray(); + this.multipleMessagesData = CreateBufferWithMultipleMessages(message, maxMessageCount); + } + + public IMessage ParseFromByteArray() => parser.ParseFrom(data); + + public void ParseDelimitedMessagesFromByteArray(int messageCount) + { + var input = new CodedInputStream(multipleMessagesData); + for (int i = 0; i < messageCount; i++) + { + var msg = factory(); + input.ReadMessage(msg); + } + } + + private static byte[] CreateBufferWithMultipleMessages(IMessage msg, int msgCount) + { + var ms = new MemoryStream(); + var cos = new CodedOutputStream(ms); + for (int i = 0; i < msgCount; i++) + { + cos.WriteMessage(msg); + } + cos.Flush(); + return ms.ToArray(); + } + } + } +} diff --git a/csharp/src/Google.Protobuf.Benchmarks/WrapperBenchmark.cs b/csharp/src/Google.Protobuf.Benchmarks/WrapperBenchmark.cs deleted file mode 100644 index ae17c1819a..0000000000 --- a/csharp/src/Google.Protobuf.Benchmarks/WrapperBenchmark.cs +++ /dev/null @@ -1,102 +0,0 @@ -#region Copyright notice and license -// Protocol Buffers - Google's data interchange format -// Copyright 2019 Google Inc. All rights reserved. -// https://github.com/protocolbuffers/protobuf -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#endregion - -using BenchmarkDotNet.Attributes; -using System.Collections.Generic; -using System.IO; -using System.Linq; - -namespace Google.Protobuf.Benchmarks -{ - /// - /// Benchmark that tests serialization/deserialization of wrapper fields. - /// - [MemoryDiagnoser] - public class WrapperBenchmark - { - byte[] manyWrapperFieldsData; - byte[] manyPrimitiveFieldsData; - - [GlobalSetup] - public void GlobalSetup() - { - manyWrapperFieldsData = CreateManyWrapperFieldsMessage().ToByteArray(); - manyPrimitiveFieldsData = CreateManyPrimitiveFieldsMessage().ToByteArray(); - } - - [Benchmark] - public ManyWrapperFieldsMessage ParseWrapperFields() - { - return ManyWrapperFieldsMessage.Parser.ParseFrom(manyWrapperFieldsData); - } - - [Benchmark] - public ManyPrimitiveFieldsMessage ParsePrimitiveFields() - { - return ManyPrimitiveFieldsMessage.Parser.ParseFrom(manyPrimitiveFieldsData); - } - - private static ManyWrapperFieldsMessage CreateManyWrapperFieldsMessage() - { - // Example data match data of an internal benchmarks - return new ManyWrapperFieldsMessage() - { - Int64Field19 = 123, - Int64Field37 = 1000032, - Int64Field26 = 3453524500, - DoubleField79 = 1.2, - DoubleField25 = 234, - DoubleField9 = 123.3, - DoubleField28 = 23, - DoubleField7 = 234, - DoubleField50 = 2.45 - }; - } - - private static ManyPrimitiveFieldsMessage CreateManyPrimitiveFieldsMessage() - { - // Example data match data of an internal benchmarks - return new ManyPrimitiveFieldsMessage() - { - Int64Field19 = 123, - Int64Field37 = 1000032, - Int64Field26 = 3453524500, - DoubleField79 = 1.2, - DoubleField25 = 234, - DoubleField9 = 123.3, - DoubleField28 = 23, - DoubleField7 = 234, - DoubleField50 = 2.45 - }; - } - } -}