Chiebot-Mirror
/
protobuf
mirror of https://github.com/protocolbuffers/protobuf.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Support creating FileDescriptors dynamically from binary data.

Browse Source 
Related to #658 and #5007.
pull/5064/head
Jon Skeet 6 years ago committed by Jon Skeet
parent fa252ec2a5
commit 228530e2da
6 changed files with 168 additions and 40 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 99
      csharp/src/Google.Protobuf.Test/Reflection/DescriptorsTest.cs
  2. 8
      csharp/src/Google.Protobuf/Reflection/DescriptorPool.cs
  3. 12
      csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs
  4. 63
      csharp/src/Google.Protobuf/Reflection/FileDescriptor.cs
  5. 10
      csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs
  6. 16
      csharp/src/Google.Protobuf/Reflection/OneofDescriptor.cs

99
csharp/src/Google.Protobuf.Test/Reflection/DescriptorsTest.cs
Unescape View File

@ -30,10 +30,11 @@
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#endregion
using System.Linq;
using Google.Protobuf.TestProtos;
using NUnit.Framework;
using UnitTest.Issues.TestProtos;
using System;
using System.Collections.Generic;
using System.Linq;
namespace Google.Protobuf.Reflection
{
@ -44,9 +45,32 @@ namespace Google.Protobuf.Reflection
public class DescriptorsTest
{
[Test]
public void FileDescriptor()
public void FileDescriptor_GeneratedCode()
{
TestFileDescriptor(
UnittestProto3Reflection.Descriptor,
UnittestImportProto3Reflection.Descriptor,
UnittestImportPublicProto3Reflection.Descriptor);
}
[Test]
public void FileDescriptor_BuildFromByteStrings()
{
// The descriptors have to be supplied in an order such that all the
// dependencies come before the descriptors depending on them.
var descriptorData = new List<ByteString>
{
UnittestImportPublicProto3Reflection.Descriptor.Proto.ToByteString(),
UnittestImportProto3Reflection.Descriptor.Proto.ToByteString(),
UnittestProto3Reflection.Descriptor.Proto.ToByteString()
};
var converted = FileDescriptor.BuildFromByteStrings(descriptorData);
Assert.AreEqual(3, converted.Count);
TestFileDescriptor(converted[2], converted[1], converted[0]);
}
private void TestFileDescriptor(FileDescriptor file, FileDescriptor importedFile, FileDescriptor importedPublicFile)
{
FileDescriptor file = UnittestProto3Reflection.Descriptor;
Assert.AreEqual("unittest_proto3.proto", file.Name);
Assert.AreEqual("protobuf_unittest3", file.Package);
@ -56,17 +80,12 @@ namespace Google.Protobuf.Reflection
// unittest_proto3.proto doesn't have any public imports, but unittest_import_proto3.proto does.
Assert.AreEqual(0, file.PublicDependencies.Count);
Assert.AreEqual(1, UnittestImportProto3Reflection.Descriptor.PublicDependencies.Count);
Assert.AreEqual(UnittestImportPublicProto3Reflection.Descriptor, UnittestImportProto3Reflection.Descriptor.PublicDependencies[0]);
Assert.AreEqual(1, importedFile.PublicDependencies.Count);
Assert.AreEqual(importedPublicFile, importedFile.PublicDependencies[0]);
Assert.AreEqual(1, file.Dependencies.Count);
Assert.AreEqual(UnittestImportProto3Reflection.Descriptor, file.Dependencies[0]);
Assert.AreEqual(importedFile, file.Dependencies[0]);
MessageDescriptor messageType = TestAllTypes.Descriptor;
Assert.AreSame(typeof(TestAllTypes), messageType.ClrType);
Assert.AreSame(TestAllTypes.Parser, messageType.Parser);
Assert.AreEqual(messageType, file.MessageTypes[0]);
Assert.AreEqual(messageType, file.FindTypeByName<MessageDescriptor>("TestAllTypes"));
Assert.Null(file.FindTypeByName<MessageDescriptor>("NoSuchType"));
Assert.Null(file.FindTypeByName<MessageDescriptor>("protobuf_unittest3.TestAllTypes"));
for (int i = 0; i < file.MessageTypes.Count; i++)
@ -77,8 +96,8 @@ namespace Google.Protobuf.Reflection
Assert.AreEqual(file.EnumTypes[0], file.FindTypeByName<EnumDescriptor>("ForeignEnum"));
Assert.Null(file.FindTypeByName<EnumDescriptor>("NoSuchType"));
Assert.Null(file.FindTypeByName<EnumDescriptor>("protobuf_unittest3.ForeignEnum"));
Assert.AreEqual(1, UnittestImportProto3Reflection.Descriptor.EnumTypes.Count);
Assert.AreEqual("ImportEnum", UnittestImportProto3Reflection.Descriptor.EnumTypes[0].Name);
Assert.AreEqual(1, importedFile.EnumTypes.Count);
Assert.AreEqual("ImportEnum", importedFile.EnumTypes[0].Name);
for (int i = 0; i < file.EnumTypes.Count; i++)
{
Assert.AreEqual(i, file.EnumTypes[i].Index);
@ -97,6 +116,56 @@ namespace Google.Protobuf.Reflection
Assert.AreEqual("protobuf_unittest", file.Package);
}
[Test]
public void FileDescriptor_BuildFromByteStrings_MissingDependency()
{
var descriptorData = new List<ByteString>
{
UnittestImportProto3Reflection.Descriptor.Proto.ToByteString(),
UnittestProto3Reflection.Descriptor.Proto.ToByteString(),
};
// This will fail, because we're missing UnittestImportPublicProto3Reflection
Assert.Throws<ArgumentException>(() => FileDescriptor.BuildFromByteStrings(descriptorData));
}
[Test]
public void FileDescriptor_BuildFromByteStrings_DuplicateNames()
{
var descriptorData = new List<ByteString>
{
UnittestImportPublicProto3Reflection.Descriptor.Proto.ToByteString(),
UnittestImportPublicProto3Reflection.Descriptor.Proto.ToByteString(),
};
// This will fail due to the same name being used twice
Assert.Throws<ArgumentException>(() => FileDescriptor.BuildFromByteStrings(descriptorData));
}
[Test]
public void FileDescriptor_BuildFromByteStrings_IncorrectOrder()
{
var descriptorData = new List<ByteString>
{
UnittestProto3Reflection.Descriptor.Proto.ToByteString(),
UnittestImportPublicProto3Reflection.Descriptor.Proto.ToByteString(),
UnittestImportProto3Reflection.Descriptor.Proto.ToByteString()
};
// This will fail, because the dependencies should come first
Assert.Throws<ArgumentException>(() => FileDescriptor.BuildFromByteStrings(descriptorData));
}
[Test]
public void MessageDescriptorFromGeneratedCodeFileDescriptor()
{
var file = UnittestProto3Reflection.Descriptor;
MessageDescriptor messageType = TestAllTypes.Descriptor;
Assert.AreSame(typeof(TestAllTypes), messageType.ClrType);
Assert.AreSame(TestAllTypes.Parser, messageType.Parser);
Assert.AreEqual(messageType, file.MessageTypes[0]);
Assert.AreEqual(messageType, file.FindTypeByName<MessageDescriptor>("TestAllTypes"));
}
[Test]
public void MessageDescriptor()
{
@ -163,7 +232,7 @@ namespace Google.Protobuf.Reflection
Assert.AreEqual("single_nested_enum", enumField.Name);
Assert.AreEqual(FieldType.Enum, enumField.FieldType);
// Assert.AreEqual(TestAllTypes.Types.NestedEnum.DescriptorProtoFile, enumField.EnumType);
Assert.AreEqual(messageType.EnumTypes[0], enumField.EnumType);
Assert.AreEqual("single_foreign_message", messageField.Name);
Assert.AreEqual(FieldType.Message, messageField.FieldType);

8
csharp/src/Google.Protobuf/Reflection/DescriptorPool.cs
Unescape View File

@ -53,13 +53,13 @@ namespace Google.Protobuf.Reflection
private readonly HashSet<FileDescriptor> dependencies;
internal DescriptorPool(FileDescriptor[] dependencyFiles)
internal DescriptorPool(IEnumerable<FileDescriptor> dependencyFiles)
{
dependencies = new HashSet<FileDescriptor>();
for (int i = 0; i < dependencyFiles.Length; i++)
foreach (var dependencyFile in dependencyFiles)
{
dependencies.Add(dependencyFiles[i]);
ImportPublicDependencies(dependencyFiles[i]);
dependencies.Add(dependencyFile);
ImportPublicDependencies(dependencyFile);
}
foreach (FileDescriptor dependency in dependencyFiles)

12
csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs
Unescape View File

@ -116,13 +116,18 @@ namespace Google.Protobuf.Reflection
/// that is the responsibility of the accessor.
/// </para>
/// <para>
/// The value returned by this property will be non-null for all regular fields. However,
/// if a message containing a map field is introspected, the list of nested messages will include
/// In descriptors for generated code, the value returned by this property will be non-null for all
/// regular fields. However, if a message containing a map field is introspected, the list of nested messages will include
/// an auto-generated nested key/value pair message for the field. This is not represented in any
/// generated type, and the value of the map field itself is represented by a dictionary in the
/// reflection API. There are never instances of those "hidden" messages, so no accessor is provided
/// and this property will return null.
/// </para>
/// <para>
/// In dynamically loaded descriptors, the value returned by this property will current be null;
/// if and when dynamic messages are supported, it will return a suitable accessor to work with
/// them.
/// </para>
/// </remarks>
public IFieldAccessor Accessor => accessor;
@ -330,7 +335,8 @@ namespace Google.Protobuf.Reflection
private IFieldAccessor CreateAccessor()
{
// If we're given no property name, that's because we really don't want an accessor.
// (At the moment, that means it's a map entry message...)
// This could be because it's a map message, or it could be that we're loading a FileDescriptor dynamically.
// TODO: Support dynamic messages.
if (propertyName == null)
{
return null;

63
csharp/src/Google.Protobuf/Reflection/FileDescriptor.cs
Unescape View File

@ -34,6 +34,7 @@ using Google.Protobuf.WellKnownTypes;
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Linq;
namespace Google.Protobuf.Reflection
{
@ -54,12 +55,12 @@ namespace Google.Protobuf.Reflection
ForceReflectionInitialization<Value.KindOneofCase>();
}
private FileDescriptor(ByteString descriptorData, FileDescriptorProto proto, FileDescriptor[] dependencies, DescriptorPool pool, bool allowUnknownDependencies, GeneratedClrTypeInfo generatedCodeInfo)
private FileDescriptor(ByteString descriptorData, FileDescriptorProto proto, IEnumerable<FileDescriptor> dependencies, DescriptorPool pool, bool allowUnknownDependencies, GeneratedClrTypeInfo generatedCodeInfo)
{
SerializedData = descriptorData;
DescriptorPool = pool;
Proto = proto;
Dependencies = new ReadOnlyCollection<FileDescriptor>((FileDescriptor[]) dependencies.Clone());
Dependencies = new ReadOnlyCollection<FileDescriptor>(dependencies.ToList());
PublicDependencies = DeterminePublicDependencies(this, proto, dependencies, allowUnknownDependencies);
@ -67,11 +68,11 @@ namespace Google.Protobuf.Reflection
MessageTypes = DescriptorUtil.ConvertAndMakeReadOnly(proto.MessageType,
(message, index) =>
new MessageDescriptor(message, this, null, index, generatedCodeInfo.NestedTypes[index]));
new MessageDescriptor(message, this, null, index, generatedCodeInfo?.NestedTypes[index]));
EnumTypes = DescriptorUtil.ConvertAndMakeReadOnly(proto.EnumType,
(enumType, index) =>
new EnumDescriptor(enumType, this, null, index, generatedCodeInfo.NestedEnums[index]));
new EnumDescriptor(enumType, this, null, index, generatedCodeInfo?.NestedEnums[index]));
Services = DescriptorUtil.ConvertAndMakeReadOnly(proto.Service,
(service, index) =>
@ -98,13 +99,9 @@ namespace Google.Protobuf.Reflection
/// Extracts public dependencies from direct dependencies. This is a static method despite its
/// first parameter, as the value we're in the middle of constructing is only used for exceptions.
/// </summary>
private static IList<FileDescriptor> DeterminePublicDependencies(FileDescriptor @this, FileDescriptorProto proto, FileDescriptor[] dependencies, bool allowUnknownDependencies)
private static IList<FileDescriptor> DeterminePublicDependencies(FileDescriptor @this, FileDescriptorProto proto, IEnumerable<FileDescriptor> dependencies, bool allowUnknownDependencies)
{
var nameToFileMap = new Dictionary<string, FileDescriptor>();
foreach (var file in dependencies)
{
nameToFileMap[file.Name] = file;
}
var nameToFileMap = dependencies.ToDictionary(file => file.Name);
var publicDependencies = new List<FileDescriptor>();
for (int i = 0; i < proto.PublicDependency.Count; i++)
{
@ -114,8 +111,7 @@ namespace Google.Protobuf.Reflection
throw new DescriptorValidationException(@this, "Invalid public dependency index.");
}
string name = proto.Dependency[index];
FileDescriptor file = nameToFileMap[name];
if (file == null)
if (!nameToFileMap.TryGetValue(name, out var file))
{
if (!allowUnknownDependencies)
{
@ -315,6 +311,49 @@ namespace Google.Protobuf.Reflection
}
}
/// <summary>
/// Converts the given descriptor binary data into FileDescriptor objects.
/// Note: reflection using the returned FileDescriptors is not currently supported.
/// </summary>
/// <param name="descriptorData">The binary file descriptor proto data. Must not be null, and any
/// dependencies must come before the descriptor which depends on them. (If A depends on B, and B
/// depends on C, then the descriptors must be presented in the order C, B, A.) This is compatible
/// with the order in which protoc provides descriptors to plugins.</param>
/// <returns>The file descriptors corresponding to <paramref name="descriptorData"/>.</returns>
public static IReadOnlyList<FileDescriptor> BuildFromByteStrings(IEnumerable<ByteString> descriptorData)
{
ProtoPreconditions.CheckNotNull(descriptorData, nameof(descriptorData));
// TODO: See if we can build a single DescriptorPool instead of building lots of them.
// This will all behave correctly, but it's less efficient than we'd like.
var descriptors = new List<FileDescriptor>();
var descriptorsByName = new Dictionary<string, FileDescriptor>();
foreach (var data in descriptorData)
{
var proto = FileDescriptorProto.Parser.ParseFrom(data);
var dependencies = new List<FileDescriptor>();
foreach (var dependencyName in proto.Dependency)
{
if (!descriptorsByName.TryGetValue(dependencyName, out var dependency))
{
throw new ArgumentException($"Dependency missing: {dependencyName}");
}
dependencies.Add(dependency);
}
var pool = new DescriptorPool(dependencies);
FileDescriptor descriptor = new FileDescriptor(
data, proto, dependencies, pool,
allowUnknownDependencies: false, generatedCodeInfo: null);
descriptors.Add(descriptor);
if (descriptorsByName.ContainsKey(descriptor.Name))
{
throw new ArgumentException($"Duplicate descriptor name: {descriptor.Name}");
}
descriptorsByName.Add(descriptor.Name, descriptor);
}
return new ReadOnlyCollection<FileDescriptor>(descriptors);
}
/// <summary>
/// Returns a <see cref="System.String" /> that represents this instance.
/// </summary>

10
csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs
Unescape View File

@ -72,23 +72,21 @@ namespace Google.Protobuf.Reflection
ClrType = generatedCodeInfo?.ClrType;
ContainingType = parent;
// Note use of generatedCodeInfo. rather than generatedCodeInfo?. here... we don't expect
// to see any nested oneofs, types or enums in "not actually generated" code... we do
// expect fields though (for map entry messages).
// If generatedCodeInfo is null, we just won't generate an accessor for any fields.
Oneofs = DescriptorUtil.ConvertAndMakeReadOnly(
proto.OneofDecl,
(oneof, index) =>
new OneofDescriptor(oneof, file, this, index, generatedCodeInfo.OneofNames[index]));
new OneofDescriptor(oneof, file, this, index, generatedCodeInfo?.OneofNames[index]));
NestedTypes = DescriptorUtil.ConvertAndMakeReadOnly(
proto.NestedType,
(type, index) =>
new MessageDescriptor(type, file, this, index, generatedCodeInfo.NestedTypes[index]));
new MessageDescriptor(type, file, this, index, generatedCodeInfo?.NestedTypes[index]));
EnumTypes = DescriptorUtil.ConvertAndMakeReadOnly(
proto.EnumType,
(type, index) =>
new EnumDescriptor(type, file, this, index, generatedCodeInfo.NestedEnums[index]));
new EnumDescriptor(type, file, this, index, generatedCodeInfo?.NestedEnums[index]));
fieldsInDeclarationOrder = DescriptorUtil.ConvertAndMakeReadOnly(
proto.Field,

16
csharp/src/Google.Protobuf/Reflection/OneofDescriptor.cs
Unescape View File

@ -85,6 +85,16 @@ namespace Google.Protobuf.Reflection
/// Gets an accessor for reflective access to the values associated with the oneof
/// in a particular message.
/// </summary>
/// <remarks>
/// <para>
/// In descriptors for generated code, the value returned by this property will always be non-null.
/// </para>
/// <para>
/// In dynamically loaded descriptors, the value returned by this property will current be null;
/// if and when dynamic messages are supported, it will return a suitable accessor to work with
/// them.
/// </para>
/// </remarks>
/// <value>
/// The accessor used for reflective access.
/// </value>
@ -110,6 +120,12 @@ namespace Google.Protobuf.Reflection
private OneofAccessor CreateAccessor(string clrName)
{
// We won't have a CLR name if this is from a dynamically-loaded FileDescriptor.
// TODO: Support dynamic messages.
if (clrName == null)
{
return null;
}
var caseProperty = containingType.ClrType.GetProperty(clrName + "Case");
if (caseProperty == null)
{

Write Preview
Loading…
Cancel
Save