Merge pull request #7351 from jtattermusch/protobuf_csharp_new_parsing
New Span-based C# parsing logicpull/7476/head
Jan Tattermusch
5 years ago
committed by
GitHub
commit
a690227398
76 changed files with 6088 additions and 2573 deletions
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,233 @@ |
||||
#region Copyright notice and license |
||||
// Protocol Buffers - Google's data interchange format |
||||
// Copyright 2008 Google Inc. All rights reserved. |
||||
// https://developers.google.com/protocol-buffers/ |
||||
// |
||||
// Redistribution and use in source and binary forms, with or without |
||||
// modification, are permitted provided that the following conditions are |
||||
// met: |
||||
// |
||||
// * Redistributions of source code must retain the above copyright |
||||
// notice, this list of conditions and the following disclaimer. |
||||
// * Redistributions in binary form must reproduce the above |
||||
// copyright notice, this list of conditions and the following disclaimer |
||||
// in the documentation and/or other materials provided with the |
||||
// distribution. |
||||
// * Neither the name of Google Inc. nor the names of its |
||||
// contributors may be used to endorse or promote products derived from |
||||
// this software without specific prior written permission. |
||||
// |
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
||||
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
||||
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
||||
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
||||
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
||||
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
||||
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
||||
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
#endregion |
||||
using Google.Protobuf; |
||||
using Google.Protobuf.Reflection; |
||||
using System.Buffers; |
||||
using pb = global::Google.Protobuf; |
||||
using pbr = global::Google.Protobuf.Reflection; |
||||
using NUnit.Framework; |
||||
|
||||
|
||||
namespace Google.Protobuf |
||||
{ |
||||
public class LegacyGeneratedCodeTest |
||||
{ |
||||
[Test] |
||||
public void IntermixingOfNewAndLegacyGeneratedCodeWorksWithCodedInputStream() |
||||
{ |
||||
var message = new ParseContextEnabledMessageB |
||||
{ |
||||
A = new LegacyGeneratedCodeMessageA |
||||
{ |
||||
Bb = new ParseContextEnabledMessageB { OptionalInt32 = 12345 } |
||||
}, |
||||
OptionalInt32 = 6789 |
||||
}; |
||||
var data = message.ToByteArray(); |
||||
|
||||
// when parsing started using CodedInputStream and a message with legacy generated code |
||||
// is encountered somewhere in the parse tree, we still need to be able to use its |
||||
// MergeFrom(CodedInputStream) method to parse correctly. |
||||
var codedInput = new CodedInputStream(data); |
||||
var parsed = new ParseContextEnabledMessageB(); |
||||
codedInput.ReadRawMessage(parsed); |
||||
Assert.IsTrue(codedInput.IsAtEnd); |
||||
|
||||
Assert.AreEqual(12345, parsed.A.Bb.OptionalInt32); |
||||
Assert.AreEqual(6789, parsed.OptionalInt32); |
||||
} |
||||
|
||||
[Test] |
||||
public void LegacyGeneratedCodeThrowsWithReadOnlySequence() |
||||
{ |
||||
var message = new ParseContextEnabledMessageB |
||||
{ |
||||
A = new LegacyGeneratedCodeMessageA |
||||
{ |
||||
Bb = new ParseContextEnabledMessageB { OptionalInt32 = 12345 } |
||||
}, |
||||
OptionalInt32 = 6789 |
||||
}; |
||||
var data = message.ToByteArray(); |
||||
|
||||
// if parsing started using ReadOnlySequence and we don't have a CodedInputStream |
||||
// instance at hand, we cannot fall back to the legacy MergeFrom(CodedInputStream) |
||||
// method and parsing will fail. As a consequence, one can only use parsing |
||||
// from ReadOnlySequence if all the messages in the parsing tree have their generated |
||||
// code up to date. |
||||
var exception = Assert.Throws<InvalidProtocolBufferException>(() => |
||||
{ |
||||
ParseContext.Initialize(new ReadOnlySequence<byte>(data), out ParseContext parseCtx); |
||||
var parsed = new ParseContextEnabledMessageB(); |
||||
ParsingPrimitivesMessages.ReadRawMessage(ref parseCtx, parsed); |
||||
}); |
||||
Assert.AreEqual($"Message {typeof(LegacyGeneratedCodeMessageA).Name} doesn't provide the generated method that enables ParseContext-based parsing. You might need to regenerate the generated protobuf code.", exception.Message); |
||||
} |
||||
|
||||
// hand-modified version of a generated message that only provides the legacy |
||||
// MergeFrom(CodedInputStream) method and doesn't implement IBufferMessage. |
||||
private sealed partial class LegacyGeneratedCodeMessageA : pb::IMessage { |
||||
private pb::UnknownFieldSet _unknownFields; |
||||
|
||||
pbr::MessageDescriptor pb::IMessage.Descriptor => throw new System.NotImplementedException(); |
||||
|
||||
/// <summary>Field number for the "bb" field.</summary> |
||||
public const int BbFieldNumber = 1; |
||||
private ParseContextEnabledMessageB bb_; |
||||
public ParseContextEnabledMessageB Bb { |
||||
get { return bb_; } |
||||
set { |
||||
bb_ = value; |
||||
} |
||||
} |
||||
|
||||
public void WriteTo(pb::CodedOutputStream output) { |
||||
if (bb_ != null) { |
||||
output.WriteRawTag(10); |
||||
output.WriteMessage(Bb); |
||||
} |
||||
if (_unknownFields != null) { |
||||
_unknownFields.WriteTo(output); |
||||
} |
||||
} |
||||
|
||||
public int CalculateSize() { |
||||
int size = 0; |
||||
if (bb_ != null) { |
||||
size += 1 + pb::CodedOutputStream.ComputeMessageSize(Bb); |
||||
} |
||||
if (_unknownFields != null) { |
||||
size += _unknownFields.CalculateSize(); |
||||
} |
||||
return size; |
||||
} |
||||
|
||||
public void MergeFrom(pb::CodedInputStream input) { |
||||
uint tag; |
||||
while ((tag = input.ReadTag()) != 0) { |
||||
switch(tag) { |
||||
default: |
||||
_unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); |
||||
break; |
||||
case 10: { |
||||
if (bb_ == null) { |
||||
Bb = new ParseContextEnabledMessageB(); |
||||
} |
||||
input.ReadMessage(Bb); |
||||
break; |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
// hand-modified version of a generated message that does provide |
||||
// the new InternalMergeFrom(ref ParseContext) method. |
||||
private sealed partial class ParseContextEnabledMessageB : pb::IBufferMessage { |
||||
private pb::UnknownFieldSet _unknownFields; |
||||
|
||||
pbr::MessageDescriptor pb::IMessage.Descriptor => throw new System.NotImplementedException(); |
||||
|
||||
/// <summary>Field number for the "a" field.</summary> |
||||
public const int AFieldNumber = 1; |
||||
private LegacyGeneratedCodeMessageA a_; |
||||
public LegacyGeneratedCodeMessageA A { |
||||
get { return a_; } |
||||
set { |
||||
a_ = value; |
||||
} |
||||
} |
||||
|
||||
/// <summary>Field number for the "optional_int32" field.</summary> |
||||
public const int OptionalInt32FieldNumber = 2; |
||||
private int optionalInt32_; |
||||
public int OptionalInt32 { |
||||
get { return optionalInt32_; } |
||||
set { |
||||
optionalInt32_ = value; |
||||
} |
||||
} |
||||
|
||||
public void WriteTo(pb::CodedOutputStream output) { |
||||
if (a_ != null) { |
||||
output.WriteRawTag(10); |
||||
output.WriteMessage(A); |
||||
} |
||||
if (OptionalInt32 != 0) { |
||||
output.WriteRawTag(16); |
||||
output.WriteInt32(OptionalInt32); |
||||
} |
||||
if (_unknownFields != null) { |
||||
_unknownFields.WriteTo(output); |
||||
} |
||||
} |
||||
public int CalculateSize() { |
||||
int size = 0; |
||||
if (a_ != null) { |
||||
size += 1 + pb::CodedOutputStream.ComputeMessageSize(A); |
||||
} |
||||
if (OptionalInt32 != 0) { |
||||
size += 1 + pb::CodedOutputStream.ComputeInt32Size(OptionalInt32); |
||||
} |
||||
if (_unknownFields != null) { |
||||
size += _unknownFields.CalculateSize(); |
||||
} |
||||
return size; |
||||
} |
||||
public void MergeFrom(pb::CodedInputStream input) { |
||||
input.ReadRawMessage(this); |
||||
} |
||||
|
||||
void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { |
||||
uint tag; |
||||
while ((tag = input.ReadTag()) != 0) { |
||||
switch(tag) { |
||||
default: |
||||
_unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); |
||||
break; |
||||
case 10: { |
||||
if (a_ == null) { |
||||
A = new LegacyGeneratedCodeMessageA(); |
||||
} |
||||
input.ReadMessage(A); |
||||
break; |
||||
} |
||||
case 16: { |
||||
OptionalInt32 = input.ReadInt32(); |
||||
break; |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,100 @@ |
||||
#region Copyright notice and license |
||||
// Protocol Buffers - Google's data interchange format |
||||
// Copyright 2015 Google Inc. All rights reserved. |
||||
// https://developers.google.com/protocol-buffers/ |
||||
// |
||||
// Redistribution and use in source and binary forms, with or without |
||||
// modification, are permitted provided that the following conditions are |
||||
// met: |
||||
// |
||||
// * Redistributions of source code must retain the above copyright |
||||
// notice, this list of conditions and the following disclaimer. |
||||
// * Redistributions in binary form must reproduce the above |
||||
// copyright notice, this list of conditions and the following disclaimer |
||||
// in the documentation and/or other materials provided with the |
||||
// distribution. |
||||
// * Neither the name of Google Inc. nor the names of its |
||||
// contributors may be used to endorse or promote products derived from |
||||
// this software without specific prior written permission. |
||||
// |
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
||||
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
||||
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
||||
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
||||
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
||||
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
||||
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
||||
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
#endregion |
||||
|
||||
using NUnit.Framework; |
||||
using System; |
||||
using System.Buffers; |
||||
|
||||
namespace Google.Protobuf |
||||
{ |
||||
public static class MessageParsingHelpers |
||||
{ |
||||
public static void AssertReadingMessage<T>(MessageParser<T> parser, byte[] bytes, Action<T> assert) where T : IMessage<T> |
||||
{ |
||||
var parsedStream = parser.ParseFrom(bytes); |
||||
|
||||
// Load content as single segment |
||||
var parsedBuffer = parser.ParseFrom(new ReadOnlySequence<byte>(bytes)); |
||||
assert(parsedBuffer); |
||||
|
||||
// Load content as multiple segments |
||||
parsedBuffer = parser.ParseFrom(ReadOnlySequenceFactory.CreateWithContent(bytes)); |
||||
assert(parsedBuffer); |
||||
|
||||
assert(parsedStream); |
||||
} |
||||
|
||||
public static void AssertReadingMessage(MessageParser parser, byte[] bytes, Action<IMessage> assert) |
||||
{ |
||||
var parsedStream = parser.ParseFrom(bytes); |
||||
|
||||
// Load content as single segment |
||||
var parsedBuffer = parser.ParseFrom(new ReadOnlySequence<byte>(bytes)); |
||||
assert(parsedBuffer); |
||||
|
||||
// Load content as multiple segments |
||||
parsedBuffer = parser.ParseFrom(ReadOnlySequenceFactory.CreateWithContent(bytes)); |
||||
assert(parsedBuffer); |
||||
|
||||
assert(parsedStream); |
||||
} |
||||
|
||||
public static void AssertReadingMessageThrows<TMessage, TException>(MessageParser<TMessage> parser, byte[] bytes) |
||||
where TMessage : IMessage<TMessage> |
||||
where TException : Exception |
||||
{ |
||||
Assert.Throws<TException>(() => parser.ParseFrom(bytes)); |
||||
|
||||
Assert.Throws<TException>(() => parser.ParseFrom(new ReadOnlySequence<byte>(bytes))); |
||||
} |
||||
|
||||
public static void AssertRoundtrip<T>(MessageParser<T> parser, T message, Action<T> additionalAssert = null) where T : IMessage<T> |
||||
{ |
||||
var bytes = message.ToByteArray(); |
||||
|
||||
// Load content as single segment |
||||
var parsedBuffer = parser.ParseFrom(new ReadOnlySequence<byte>(bytes)); |
||||
Assert.AreEqual(message, parsedBuffer); |
||||
additionalAssert?.Invoke(parsedBuffer); |
||||
|
||||
// Load content as multiple segments |
||||
parsedBuffer = parser.ParseFrom(ReadOnlySequenceFactory.CreateWithContent(bytes)); |
||||
Assert.AreEqual(message, parsedBuffer); |
||||
additionalAssert?.Invoke(parsedBuffer); |
||||
|
||||
var parsedStream = parser.ParseFrom(bytes); |
||||
|
||||
Assert.AreEqual(message, parsedStream); |
||||
additionalAssert?.Invoke(parsedStream); |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,128 @@ |
||||
#region Copyright notice and license |
||||
// Protocol Buffers - Google's data interchange format |
||||
// Copyright 2008 Google Inc. All rights reserved. |
||||
// https://developers.google.com/protocol-buffers/ |
||||
// |
||||
// Redistribution and use in source and binary forms, with or without |
||||
// modification, are permitted provided that the following conditions are |
||||
// met: |
||||
// |
||||
// * Redistributions of source code must retain the above copyright |
||||
// notice, this list of conditions and the following disclaimer. |
||||
// * Redistributions in binary form must reproduce the above |
||||
// copyright notice, this list of conditions and the following disclaimer |
||||
// in the documentation and/or other materials provided with the |
||||
// distribution. |
||||
// * Neither the name of Google Inc. nor the names of its |
||||
// contributors may be used to endorse or promote products derived from |
||||
// this software without specific prior written permission. |
||||
// |
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
||||
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
||||
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
||||
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
||||
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
||||
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
||||
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
||||
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
#endregion |
||||
|
||||
using System; |
||||
using System.Buffers; |
||||
using System.Collections.Generic; |
||||
using System.Linq; |
||||
using System.Text; |
||||
using System.Threading.Tasks; |
||||
|
||||
namespace Google.Protobuf |
||||
{ |
||||
internal static class ReadOnlySequenceFactory |
||||
{ |
||||
public static ReadOnlySequence<byte> CreateWithContent(byte[] data, int segmentSize = 1) |
||||
{ |
||||
var segments = new List<byte[]>(); |
||||
|
||||
segments.Add(new byte[0]); |
||||
var currentIndex = 0; |
||||
while (currentIndex < data.Length) |
||||
{ |
||||
var segment = new List<byte>(); |
||||
for (; currentIndex < Math.Min(currentIndex + segmentSize, data.Length); currentIndex++) |
||||
{ |
||||
segment.Add(data[currentIndex]); |
||||
} |
||||
segments.Add(segment.ToArray()); |
||||
segments.Add(new byte[0]); |
||||
} |
||||
|
||||
return CreateSegments(segments.ToArray()); |
||||
} |
||||
|
||||
/// <summary> |
||||
/// Originally from corefx, and has been contributed to Protobuf |
||||
/// https://github.com/dotnet/corefx/blob/e99ec129cfd594d53f4390bf97d1d736cff6f860/src/System.Memory/tests/ReadOnlyBuffer/ReadOnlySequenceFactory.byte.cs |
||||
/// </summary> |
||||
private static ReadOnlySequence<byte> CreateSegments(params byte[][] inputs) |
||||
{ |
||||
if (inputs == null || inputs.Length == 0) |
||||
{ |
||||
throw new InvalidOperationException(); |
||||
} |
||||
|
||||
int i = 0; |
||||
|
||||
BufferSegment last = null; |
||||
BufferSegment first = null; |
||||
|
||||
do |
||||
{ |
||||
byte[] s = inputs[i]; |
||||
int length = s.Length; |
||||
int dataOffset = length; |
||||
var chars = new byte[length * 2]; |
||||
|
||||
for (int j = 0; j < length; j++) |
||||
{ |
||||
chars[dataOffset + j] = s[j]; |
||||
} |
||||
|
||||
// Create a segment that has offset relative to the OwnedMemory and OwnedMemory itself has offset relative to array |
||||
var memory = new Memory<byte>(chars).Slice(length, length); |
||||
|
||||
if (first == null) |
||||
{ |
||||
first = new BufferSegment(memory); |
||||
last = first; |
||||
} |
||||
else |
||||
{ |
||||
last = last.Append(memory); |
||||
} |
||||
i++; |
||||
} while (i < inputs.Length); |
||||
|
||||
return new ReadOnlySequence<byte>(first, 0, last, last.Memory.Length); |
||||
} |
||||
|
||||
private class BufferSegment : ReadOnlySequenceSegment<byte> |
||||
{ |
||||
public BufferSegment(Memory<byte> memory) |
||||
{ |
||||
Memory = memory; |
||||
} |
||||
|
||||
public BufferSegment Append(Memory<byte> memory) |
||||
{ |
||||
var segment = new BufferSegment(memory) |
||||
{ |
||||
RunningIndex = RunningIndex + Memory.Length |
||||
}; |
||||
Next = segment; |
||||
return segment; |
||||
} |
||||
} |
||||
} |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in new issue