diff --git a/Makefile.am b/Makefile.am index 82ce190dec..e4dce6e5b2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -505,6 +505,7 @@ python_EXTRA_DIST= \ python/google/protobuf/internal/factory_test1.proto \ python/google/protobuf/internal/factory_test2.proto \ python/google/protobuf/internal/generator_test.py \ + python/google/protobuf/internal/json_format_test.py \ python/google/protobuf/internal/message_factory_test.py \ python/google/protobuf/internal/message_listener.py \ python/google/protobuf/internal/message_set_extensions.proto \ @@ -560,6 +561,7 @@ python_EXTRA_DIST= \ python/google/protobuf/descriptor.py \ python/google/protobuf/descriptor_database.py \ python/google/protobuf/descriptor_pool.py \ + python/google/protobuf/json_format.py \ python/google/protobuf/message.py \ python/google/protobuf/message_factory.py \ python/google/protobuf/proto_builder.py \ diff --git a/appveyor.yml b/appveyor.yml index 3447602e31..c84ecae2f3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,32 +1,32 @@ -# Only test one combination: "Visual Studio 12 + Win64 + Debug + DLL". We can -# test more combinations but AppVeyor just takes too long to finish (each -# combination takes ~15mins). -platform: - - Win64 - -configuration: - - Debug - -environment: - matrix: - - language: cpp - BUILD_DLL: ON - - - language: csharp - -install: - - ps: Start-FileDownload https://googlemock.googlecode.com/files/gmock-1.7.0.zip - - 7z x gmock-1.7.0.zip - - rename gmock-1.7.0 gmock - -before_build: - - if %platform%==Win32 set generator=Visual Studio 12 - - if %platform%==Win64 set generator=Visual Studio 12 Win64 - - if %platform%==Win32 set vcplatform=Win32 - - if %platform%==Win64 set vcplatform=x64 - -build_script: - - CALL appveyor.bat - -skip_commits: - message: /.*\[skip appveyor\].*/ +# Only test one combination: "Visual Studio 12 + Win64 + Debug + DLL". We can +# test more combinations but AppVeyor just takes too long to finish (each +# combination takes ~15mins). +platform: + - Win64 + +configuration: + - Debug + +environment: + matrix: + - language: cpp + BUILD_DLL: ON + + - language: csharp + +install: + - ps: Start-FileDownload https://googlemock.googlecode.com/files/gmock-1.7.0.zip + - 7z x gmock-1.7.0.zip + - rename gmock-1.7.0 gmock + +before_build: + - if %platform%==Win32 set generator=Visual Studio 12 + - if %platform%==Win64 set generator=Visual Studio 12 Win64 + - if %platform%==Win32 set vcplatform=Win32 + - if %platform%==Win64 set vcplatform=x64 + +build_script: + - CALL appveyor.bat + +skip_commits: + message: /.*\[skip appveyor\].*/ diff --git a/java/src/main/java/com/google/protobuf/GeneratedMessageLite.java b/java/src/main/java/com/google/protobuf/GeneratedMessageLite.java index a535b71833..4316efeef5 100644 --- a/java/src/main/java/com/google/protobuf/GeneratedMessageLite.java +++ b/java/src/main/java/com/google/protobuf/GeneratedMessageLite.java @@ -62,9 +62,8 @@ public abstract class GeneratedMessageLite< private static final long serialVersionUID = 1L; - /** For use by generated code only. */ - protected UnknownFieldSetLite unknownFields = - UnknownFieldSetLite.getDefaultInstance(); + /** For use by generated code only. Lazily initialized to reduce allocations. */ + protected UnknownFieldSetLite unknownFields = null; /** For use by generated code only. */ protected int memoizedSerializedSize = -1; @@ -84,19 +83,56 @@ public abstract class GeneratedMessageLite< return (BuilderType) dynamicMethod(MethodToInvoke.NEW_BUILDER); } + // The general strategy for unknown fields is to use an UnknownFieldSetLite that is treated as + // mutable during the parsing constructor and immutable after. This allows us to avoid + // any unnecessary intermediary allocations while reducing the generated code size. + + /** + * Lazily initializes unknown fields. + */ + private final void ensureUnknownFieldsInitialized() { + if (unknownFields == null) { + unknownFields = UnknownFieldSetLite.newInstance(); + } + } + /** - * Called by subclasses to parse an unknown field. For use by generated code - * only. + * Called by subclasses to parse an unknown field. For use by generated code only. + * * @return {@code true} unless the tag is an end-group tag. */ - protected static boolean parseUnknownField( - CodedInputStream input, - UnknownFieldSetLite.Builder unknownFields, - ExtensionRegistryLite extensionRegistry, - int tag) throws IOException { + protected boolean parseUnknownField(int tag, CodedInputStream input) throws IOException { + ensureUnknownFieldsInitialized(); return unknownFields.mergeFieldFrom(tag, input); } + /** + * Called by subclasses to parse an unknown field. For use by generated code only. + */ + protected void mergeVarintField(int tag, int value) { + ensureUnknownFieldsInitialized(); + unknownFields.mergeVarintField(tag, value); + } + + /** + * Called by subclasses to parse an unknown field. For use by generated code only. + */ + protected void mergeLengthDelimitedField(int fieldNumber, ByteString value) { + ensureUnknownFieldsInitialized(); + unknownFields.mergeLengthDelimitedField(fieldNumber, value); + } + + /** + * Called by subclasses to complete parsing. For use by generated code only. + */ + protected void doneParsing() { + if (unknownFields == null) { + unknownFields = UnknownFieldSetLite.getDefaultInstance(); + } else { + unknownFields.makeImmutable(); + } + } + public final boolean isInitialized() { return dynamicMethod(MethodToInvoke.IS_INITIALIZED, Boolean.TRUE) != null; } @@ -171,7 +207,7 @@ public abstract class GeneratedMessageLite< *

For use by generated code only. */ protected final void mergeUnknownFields(UnknownFieldSetLite unknownFields) { - this.unknownFields = UnknownFieldSetLite.concat(this.unknownFields, unknownFields); + this.unknownFields = UnknownFieldSetLite.mutableCopyOf(this.unknownFields, unknownFields); } @SuppressWarnings("unchecked") @@ -225,7 +261,13 @@ public abstract class GeneratedMessageLite< //@Override (Java 1.6 override semantics, but we must support 1.5) public MessageType buildPartial() { + if (isBuilt) { + return instance; + } + instance.dynamicMethod(MethodToInvoke.MAKE_IMMUTABLE); + instance.unknownFields.makeImmutable(); + isBuilt = true; return instance; } @@ -249,18 +291,6 @@ public abstract class GeneratedMessageLite< public MessageType getDefaultInstanceForType() { return defaultInstance; } - - /** - * Called by subclasses to parse an unknown field. - * @return {@code true} unless the tag is an end-group tag. - */ - protected boolean parseUnknownField( - CodedInputStream input, - UnknownFieldSetLite.Builder unknownFields, - ExtensionRegistryLite extensionRegistry, - int tag) throws IOException { - return unknownFields.mergeFieldFrom(tag, input); - } public BuilderType mergeFrom( com.google.protobuf.CodedInputStream input, @@ -334,6 +364,130 @@ public abstract class GeneratedMessageLite< extensions.mergeFrom(((ExtendableMessage) other).extensions); } + /** + * Parse an unknown field or an extension. For use by generated code only. + * + *

For use by generated code only. + * + * @return {@code true} unless the tag is an end-group tag. + */ + protected boolean parseUnknownField( + MessageType defaultInstance, + CodedInputStream input, + ExtensionRegistryLite extensionRegistry, + int tag) throws IOException { + int wireType = WireFormat.getTagWireType(tag); + int fieldNumber = WireFormat.getTagFieldNumber(tag); + + // TODO(dweis): How much bytecode would be saved by not requiring the generated code to + // provide the default instance? + GeneratedExtension extension = extensionRegistry.findLiteExtensionByNumber( + defaultInstance, fieldNumber); + + boolean unknown = false; + boolean packed = false; + if (extension == null) { + unknown = true; // Unknown field. + } else if (wireType == FieldSet.getWireFormatForFieldType( + extension.descriptor.getLiteType(), + false /* isPacked */)) { + packed = false; // Normal, unpacked value. + } else if (extension.descriptor.isRepeated && + extension.descriptor.type.isPackable() && + wireType == FieldSet.getWireFormatForFieldType( + extension.descriptor.getLiteType(), + true /* isPacked */)) { + packed = true; // Packed value. + } else { + unknown = true; // Wrong wire type. + } + + if (unknown) { // Unknown field or wrong wire type. Skip. + return parseUnknownField(tag, input); + } + + if (packed) { + int length = input.readRawVarint32(); + int limit = input.pushLimit(length); + if (extension.descriptor.getLiteType() == WireFormat.FieldType.ENUM) { + while (input.getBytesUntilLimit() > 0) { + int rawValue = input.readEnum(); + Object value = + extension.descriptor.getEnumType().findValueByNumber(rawValue); + if (value == null) { + // If the number isn't recognized as a valid value for this + // enum, drop it (don't even add it to unknownFields). + return true; + } + extensions.addRepeatedField(extension.descriptor, + extension.singularToFieldSetType(value)); + } + } else { + while (input.getBytesUntilLimit() > 0) { + Object value = + FieldSet.readPrimitiveField(input, + extension.descriptor.getLiteType(), + /*checkUtf8=*/ false); + extensions.addRepeatedField(extension.descriptor, value); + } + } + input.popLimit(limit); + } else { + Object value; + switch (extension.descriptor.getLiteJavaType()) { + case MESSAGE: { + MessageLite.Builder subBuilder = null; + if (!extension.descriptor.isRepeated()) { + MessageLite existingValue = + (MessageLite) extensions.getField(extension.descriptor); + if (existingValue != null) { + subBuilder = existingValue.toBuilder(); + } + } + if (subBuilder == null) { + subBuilder = extension.getMessageDefaultInstance() + .newBuilderForType(); + } + if (extension.descriptor.getLiteType() == + WireFormat.FieldType.GROUP) { + input.readGroup(extension.getNumber(), + subBuilder, extensionRegistry); + } else { + input.readMessage(subBuilder, extensionRegistry); + } + value = subBuilder.build(); + break; + } + case ENUM: + int rawValue = input.readEnum(); + value = extension.descriptor.getEnumType() + .findValueByNumber(rawValue); + // If the number isn't recognized as a valid value for this enum, + // write it to unknown fields object. + if (value == null) { + mergeVarintField(fieldNumber, rawValue); + return true; + } + break; + default: + value = FieldSet.readPrimitiveField(input, + extension.descriptor.getLiteType(), + /*checkUtf8=*/ false); + break; + } + + if (extension.descriptor.isRepeated()) { + extensions.addRepeatedField(extension.descriptor, + extension.singularToFieldSetType(value)); + } else { + extensions.setField(extension.descriptor, + extension.singularToFieldSetType(value)); + } + } + + return true; + } + private void verifyExtensionContainingType( final GeneratedExtension extension) { if (extension.getContainingTypeDefaultInstance() != @@ -404,11 +558,10 @@ public abstract class GeneratedMessageLite< } - /** - * Used by parsing constructors in generated classes. - */ - protected static void makeExtensionsImmutable( - FieldSet extensions) { + @Override + protected final void doneParsing() { + super.doneParsing(); + extensions.makeImmutable(); } @@ -619,131 +772,6 @@ public abstract class GeneratedMessageLite< } } - //----------------------------------------------------------------- - - /** - * Parse an unknown field or an extension. For use by generated code only. - * @return {@code true} unless the tag is an end-group tag. - */ - protected static - boolean parseUnknownField( - FieldSet extensions, - MessageType defaultInstance, - CodedInputStream input, - UnknownFieldSetLite.Builder unknownFields, - ExtensionRegistryLite extensionRegistry, - int tag) throws IOException { - int wireType = WireFormat.getTagWireType(tag); - int fieldNumber = WireFormat.getTagFieldNumber(tag); - - GeneratedExtension extension = - extensionRegistry.findLiteExtensionByNumber( - defaultInstance, fieldNumber); - - boolean unknown = false; - boolean packed = false; - if (extension == null) { - unknown = true; // Unknown field. - } else if (wireType == FieldSet.getWireFormatForFieldType( - extension.descriptor.getLiteType(), - false /* isPacked */)) { - packed = false; // Normal, unpacked value. - } else if (extension.descriptor.isRepeated && - extension.descriptor.type.isPackable() && - wireType == FieldSet.getWireFormatForFieldType( - extension.descriptor.getLiteType(), - true /* isPacked */)) { - packed = true; // Packed value. - } else { - unknown = true; // Wrong wire type. - } - - if (unknown) { // Unknown field or wrong wire type. Skip. - return unknownFields.mergeFieldFrom(tag, input); - } - - if (packed) { - int length = input.readRawVarint32(); - int limit = input.pushLimit(length); - if (extension.descriptor.getLiteType() == WireFormat.FieldType.ENUM) { - while (input.getBytesUntilLimit() > 0) { - int rawValue = input.readEnum(); - Object value = - extension.descriptor.getEnumType().findValueByNumber(rawValue); - if (value == null) { - // If the number isn't recognized as a valid value for this - // enum, drop it (don't even add it to unknownFields). - return true; - } - extensions.addRepeatedField(extension.descriptor, - extension.singularToFieldSetType(value)); - } - } else { - while (input.getBytesUntilLimit() > 0) { - Object value = - FieldSet.readPrimitiveField(input, - extension.descriptor.getLiteType(), - /*checkUtf8=*/ false); - extensions.addRepeatedField(extension.descriptor, value); - } - } - input.popLimit(limit); - } else { - Object value; - switch (extension.descriptor.getLiteJavaType()) { - case MESSAGE: { - MessageLite.Builder subBuilder = null; - if (!extension.descriptor.isRepeated()) { - MessageLite existingValue = - (MessageLite) extensions.getField(extension.descriptor); - if (existingValue != null) { - subBuilder = existingValue.toBuilder(); - } - } - if (subBuilder == null) { - subBuilder = extension.getMessageDefaultInstance() - .newBuilderForType(); - } - if (extension.descriptor.getLiteType() == - WireFormat.FieldType.GROUP) { - input.readGroup(extension.getNumber(), - subBuilder, extensionRegistry); - } else { - input.readMessage(subBuilder, extensionRegistry); - } - value = subBuilder.build(); - break; - } - case ENUM: - int rawValue = input.readEnum(); - value = extension.descriptor.getEnumType() - .findValueByNumber(rawValue); - // If the number isn't recognized as a valid value for this enum, - // write it to unknown fields object. - if (value == null) { - unknownFields.mergeVarintField(fieldNumber, rawValue); - return true; - } - break; - default: - value = FieldSet.readPrimitiveField(input, - extension.descriptor.getLiteType(), - /*checkUtf8=*/ false); - break; - } - - if (extension.descriptor.isRepeated()) { - extensions.addRepeatedField(extension.descriptor, - extension.singularToFieldSetType(value)); - } else { - extensions.setField(extension.descriptor, - extension.singularToFieldSetType(value)); - } - } - - return true; - } - // ----------------------------------------------------------------- /** For use by generated code only. */ @@ -893,7 +921,7 @@ public abstract class GeneratedMessageLite< extends ExtensionLite { /** - * Create a new isntance with the given parameters. + * Create a new instance with the given parameters. * * The last parameter {@code singularType} is only needed for enum types. * We store integer values for enum types in a {@link ExtendableMessage} @@ -905,7 +933,7 @@ public abstract class GeneratedMessageLite< final Type defaultValue, final MessageLite messageDefaultInstance, final ExtensionDescriptor descriptor, - Class singularType) { + final Class singularType) { // Defensive checks to verify the correct initialization order of // GeneratedExtensions and their related GeneratedMessages. if (containingTypeDefaultInstance == null) { @@ -921,24 +949,12 @@ public abstract class GeneratedMessageLite< this.defaultValue = defaultValue; this.messageDefaultInstance = messageDefaultInstance; this.descriptor = descriptor; - - // Use Java reflection to invoke the static method {@code valueOf} of - // enum types in order to convert integers to concrete enum objects. - this.singularType = singularType; - if (Internal.EnumLite.class.isAssignableFrom(singularType)) { - this.enumValueOf = getMethodOrDie( - singularType, "valueOf", int.class); - } else { - this.enumValueOf = null; - } } final ContainingType containingTypeDefaultInstance; final Type defaultValue; final MessageLite messageDefaultInstance; final ExtensionDescriptor descriptor; - final Class singularType; - final Method enumValueOf; /** * Default instance of the type being extended, used to identify that type. @@ -980,7 +996,7 @@ public abstract class GeneratedMessageLite< Object singularFromFieldSetType(final Object value) { if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) { - return invokeOrDie(enumValueOf, null, (Integer) value); + return descriptor.enumTypeMap.findValueByNumber((Integer) value); } else { return value; } diff --git a/java/src/main/java/com/google/protobuf/TextFormat.java b/java/src/main/java/com/google/protobuf/TextFormat.java index b4f4ce78b1..44d036c1cd 100644 --- a/java/src/main/java/com/google/protobuf/TextFormat.java +++ b/java/src/main/java/com/google/protobuf/TextFormat.java @@ -118,6 +118,21 @@ public final class TextFormat { } } + /** + * Generates a human readable form of the field, useful for debugging + * and other purposes, with no newline characters. + */ + public static String shortDebugString(final FieldDescriptor field, + final Object value) { + try { + final StringBuilder sb = new StringBuilder(); + SINGLE_LINE_PRINTER.printField(field, value, new TextGenerator(sb)); + return sb.toString().trim(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + /** * Generates a human readable form of the unknown fields, useful for debugging * and other purposes, with no newline characters. diff --git a/java/src/main/java/com/google/protobuf/UnknownFieldSetLite.java b/java/src/main/java/com/google/protobuf/UnknownFieldSetLite.java index 45d5fc357f..435ad4d42a 100644 --- a/java/src/main/java/com/google/protobuf/UnknownFieldSetLite.java +++ b/java/src/main/java/com/google/protobuf/UnknownFieldSetLite.java @@ -45,12 +45,13 @@ import java.util.Arrays; * @author dweis@google.com (Daniel Weis) */ public final class UnknownFieldSetLite { - - private static final int[] EMPTY_INT_ARRAY = new int[0]; - private static final Object[] EMPTY_OBJECT_ARRAY = new Object[0]; + + // Arbitrarily chosen. + // TODO(dweis): Tune this number? + private static final int MIN_CAPACITY = 8; private static final UnknownFieldSetLite DEFAULT_INSTANCE = - new UnknownFieldSetLite(0, EMPTY_INT_ARRAY, EMPTY_OBJECT_ARRAY); + new UnknownFieldSetLite(0, new int[0], new Object[0], false /* isMutable */); /** * Get an empty {@code UnknownFieldSetLite}. @@ -62,25 +63,32 @@ public final class UnknownFieldSetLite { } /** - * Create a new {@link Builder}. + * Returns an empty {@code UnknownFieldSetLite.Builder}. * *

For use by generated code only. */ public static Builder newBuilder() { return new Builder(); } + + /** + * Returns a new mutable instance. + */ + static UnknownFieldSetLite newInstance() { + return new UnknownFieldSetLite(); + } /** - * Returns an {@code UnknownFieldSetLite} that is the composite of {@code first} and + * Returns a mutable {@code UnknownFieldSetLite} that is the composite of {@code first} and * {@code second}. */ - static UnknownFieldSetLite concat(UnknownFieldSetLite first, UnknownFieldSetLite second) { + static UnknownFieldSetLite mutableCopyOf(UnknownFieldSetLite first, UnknownFieldSetLite second) { int count = first.count + second.count; int[] tags = Arrays.copyOf(first.tags, count); System.arraycopy(second.tags, 0, tags, first.count, second.count); Object[] objects = Arrays.copyOf(first.objects, count); System.arraycopy(second.objects, 0, objects, first.count, second.count); - return new UnknownFieldSetLite(count, tags, objects); + return new UnknownFieldSetLite(count, tags, objects, true /* isMutable */); } /** @@ -102,14 +110,45 @@ public final class UnknownFieldSetLite { * The lazily computed serialized size of the set. */ private int memoizedSerializedSize = -1; + + /** + * Indicates that this object is mutable. + */ + private boolean isMutable; + /** + * Constructs a mutable {@code UnknownFieldSetLite}. + */ + private UnknownFieldSetLite() { + this(0, new int[MIN_CAPACITY], new Object[MIN_CAPACITY], true /* isMutable */); + } + /** * Constructs the {@code UnknownFieldSetLite}. */ - private UnknownFieldSetLite(int count, int[] tags, Object[] objects) { + private UnknownFieldSetLite(int count, int[] tags, Object[] objects, boolean isMutable) { this.count = count; this.tags = tags; this.objects = objects; + this.isMutable = isMutable; + } + + /** + * Marks this object as immutable. + * + *

Future calls to methods that attempt to modify this object will throw. + */ + public void makeImmutable() { + this.isMutable = false; + } + + /** + * Throws an {@link UnsupportedOperationException} if immutable. + */ + void checkMutable() { + if (!isMutable) { + throw new UnsupportedOperationException(); + } } /** @@ -223,6 +262,114 @@ public final class UnknownFieldSetLite { return hashCode; } + private void storeField(int tag, Object value) { + ensureCapacity(); + + tags[count] = tag; + objects[count] = value; + count++; + } + + /** + * Ensures that our arrays are long enough to store more metadata. + */ + private void ensureCapacity() { + if (count == tags.length) { + int increment = count < (MIN_CAPACITY / 2) ? MIN_CAPACITY : count >> 1; + int newLength = count + increment; + + tags = Arrays.copyOf(tags, newLength); + objects = Arrays.copyOf(objects, newLength); + } + } + + /** + * Parse a single field from {@code input} and merge it into this set. + * + *

For use by generated code only. + * + * @param tag The field's tag number, which was already parsed. + * @return {@code false} if the tag is an end group tag. + */ + boolean mergeFieldFrom(final int tag, final CodedInputStream input) throws IOException { + checkMutable(); + final int fieldNumber = WireFormat.getTagFieldNumber(tag); + switch (WireFormat.getTagWireType(tag)) { + case WireFormat.WIRETYPE_VARINT: + storeField(tag, input.readInt64()); + return true; + case WireFormat.WIRETYPE_FIXED32: + storeField(tag, input.readFixed32()); + return true; + case WireFormat.WIRETYPE_FIXED64: + storeField(tag, input.readFixed64()); + return true; + case WireFormat.WIRETYPE_LENGTH_DELIMITED: + storeField(tag, input.readBytes()); + return true; + case WireFormat.WIRETYPE_START_GROUP: + final UnknownFieldSetLite subFieldSet = new UnknownFieldSetLite(); + subFieldSet.mergeFrom(input); + input.checkLastTagWas( + WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); + storeField(tag, subFieldSet); + return true; + case WireFormat.WIRETYPE_END_GROUP: + return false; + default: + throw InvalidProtocolBufferException.invalidWireType(); + } + } + + /** + * Convenience method for merging a new field containing a single varint + * value. This is used in particular when an unknown enum value is + * encountered. + * + *

For use by generated code only. + */ + UnknownFieldSetLite mergeVarintField(int fieldNumber, int value) { + checkMutable(); + if (fieldNumber == 0) { + throw new IllegalArgumentException("Zero is not a valid field number."); + } + + storeField(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_VARINT), (long) value); + + return this; + } + + /** + * Convenience method for merging a length-delimited field. + * + *

For use by generated code only. + */ + UnknownFieldSetLite mergeLengthDelimitedField(final int fieldNumber, final ByteString value) { + checkMutable(); + if (fieldNumber == 0) { + throw new IllegalArgumentException("Zero is not a valid field number."); + } + + storeField(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED), value); + + return this; + } + + /** + * Parse an entire message from {@code input} and merge its fields into + * this set. + */ + private UnknownFieldSetLite mergeFrom(final CodedInputStream input) throws IOException { + // Ensures initialization in mergeFieldFrom. + while (true) { + final int tag = input.readTag(); + if (tag == 0 || !mergeFieldFrom(tag, input)) { + break; + } + } + return this; + } + /** * Builder for {@link UnknownFieldSetLite}s. * @@ -230,53 +377,26 @@ public final class UnknownFieldSetLite { * *

For use by generated code only. */ + // TODO(dweis): Update the mutable API to no longer need this builder and delete. public static final class Builder { - - // Arbitrarily chosen. - // TODO(dweis): Tune this number? - private static final int MIN_CAPACITY = 8; - - private int count = 0; - private int[] tags = EMPTY_INT_ARRAY; - private Object[] objects = EMPTY_OBJECT_ARRAY; - private boolean built; - - /** - * Constructs a {@code Builder}. - */ - private Builder() {} + private UnknownFieldSetLite set; + + private Builder() { + this.set = null; + } /** * Ensures internal state is initialized for use. */ private void ensureNotBuilt() { - if (built) { - throw new IllegalStateException("Do not reuse UnknownFieldSetLite Builders."); + if (set == null) { + set = new UnknownFieldSetLite(); } - } - - private void storeField(int tag, Object value) { - ensureCapacity(); - tags[count] = tag; - objects[count] = value; - count++; + set.checkMutable(); } - /** - * Ensures that our arrays are long enough to store more metadata. - */ - private void ensureCapacity() { - if (count == tags.length) { - int increment = count < (MIN_CAPACITY / 2) ? MIN_CAPACITY : count >> 1; - int newLength = count + increment; - - tags = Arrays.copyOf(tags, newLength); - objects = Arrays.copyOf(objects, newLength); - } - } - /** * Parse a single field from {@code input} and merge it into this set. * @@ -285,36 +405,9 @@ public final class UnknownFieldSetLite { * @param tag The field's tag number, which was already parsed. * @return {@code false} if the tag is an end group tag. */ - public boolean mergeFieldFrom(final int tag, final CodedInputStream input) - throws IOException { + boolean mergeFieldFrom(final int tag, final CodedInputStream input) throws IOException { ensureNotBuilt(); - - final int fieldNumber = WireFormat.getTagFieldNumber(tag); - switch (WireFormat.getTagWireType(tag)) { - case WireFormat.WIRETYPE_VARINT: - storeField(tag, input.readInt64()); - return true; - case WireFormat.WIRETYPE_FIXED32: - storeField(tag, input.readFixed32()); - return true; - case WireFormat.WIRETYPE_FIXED64: - storeField(tag, input.readFixed64()); - return true; - case WireFormat.WIRETYPE_LENGTH_DELIMITED: - storeField(tag, input.readBytes()); - return true; - case WireFormat.WIRETYPE_START_GROUP: - final Builder subBuilder = newBuilder(); - subBuilder.mergeFrom(input); - input.checkLastTagWas( - WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); - storeField(tag, subBuilder.build()); - return true; - case WireFormat.WIRETYPE_END_GROUP: - return false; - default: - throw InvalidProtocolBufferException.invalidWireType(); - } + return set.mergeFieldFrom(tag, input); } /** @@ -324,71 +417,42 @@ public final class UnknownFieldSetLite { * *

For use by generated code only. */ - public Builder mergeVarintField(int fieldNumber, int value) { - if (fieldNumber == 0) { - throw new IllegalArgumentException("Zero is not a valid field number."); - } + Builder mergeVarintField(int fieldNumber, int value) { ensureNotBuilt(); - - storeField(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_VARINT), (long) value); - + set.mergeVarintField(fieldNumber, value); return this; } - + /** * Convenience method for merging a length-delimited field. * *

For use by generated code only. */ - public Builder mergeLengthDelimitedField( - final int fieldNumber, final ByteString value) { - if (fieldNumber == 0) { - throw new IllegalArgumentException("Zero is not a valid field number."); - } - ensureNotBuilt(); - - storeField(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_LENGTH_DELIMITED), value); - + public Builder mergeLengthDelimitedField(final int fieldNumber, final ByteString value) { + ensureNotBuilt(); + set.mergeLengthDelimitedField(fieldNumber, value); return this; } - /** - * Parse an entire message from {@code input} and merge its fields into - * this set. - */ - private Builder mergeFrom(final CodedInputStream input) throws IOException { - // Ensures initialization in mergeFieldFrom. - while (true) { - final int tag = input.readTag(); - if (tag == 0 || !mergeFieldFrom(tag, input)) { - break; - } - } - return this; - } - /** * Build the {@link UnknownFieldSetLite} and return it. * *

Once {@code build()} has been called, the {@code Builder} will no * longer be usable. Calling any method after {@code build()} will result - * in undefined behavior and can cause a {@code IllegalStateException} to be - * thrown. + * in undefined behavior and can cause an + * {@code UnsupportedOperationException} to be thrown. * *

For use by generated code only. */ public UnknownFieldSetLite build() { - if (built) { - throw new IllegalStateException("Do not reuse UnknownFieldSetLite Builders."); - } - - built = true; - - if (count == 0) { + if (set == null) { return DEFAULT_INSTANCE; } + + set.checkMutable(); + set.makeImmutable(); - return new UnknownFieldSetLite(count, tags, objects); + return set; } } } diff --git a/java/src/main/java/com/google/protobuf/Utf8.java b/java/src/main/java/com/google/protobuf/Utf8.java index 0699778f73..48c7e9e62d 100644 --- a/java/src/main/java/com/google/protobuf/Utf8.java +++ b/java/src/main/java/com/google/protobuf/Utf8.java @@ -360,8 +360,8 @@ final class Utf8 { static class UnpairedSurrogateException extends IllegalArgumentException { - private UnpairedSurrogateException(int index) { - super("Unpaired surrogate at index " + index); + private UnpairedSurrogateException(int index, int length) { + super("Unpaired surrogate at index " + index + " of " + length); } } @@ -417,7 +417,7 @@ final class Utf8 { // Check that we have a well-formed surrogate pair. int cp = Character.codePointAt(sequence, i); if (cp < Character.MIN_SUPPLEMENTARY_CODE_POINT) { - throw new UnpairedSurrogateException(i); + throw new UnpairedSurrogateException(i, utf16Length); } i++; } @@ -457,7 +457,7 @@ final class Utf8 { final char low; if (i + 1 == sequence.length() || !Character.isSurrogatePair(c, (low = sequence.charAt(++i)))) { - throw new UnpairedSurrogateException((i - 1)); + throw new UnpairedSurrogateException((i - 1), utf16Length); } int codePoint = Character.toCodePoint(c, low); bytes[j++] = (byte) ((0xF << 4) | (codePoint >>> 18)); @@ -470,7 +470,7 @@ final class Utf8 { if ((Character.MIN_SURROGATE <= c && c <= Character.MAX_SURROGATE) && (i + 1 == sequence.length() || !Character.isSurrogatePair(c, sequence.charAt(i + 1)))) { - throw new UnpairedSurrogateException(i); + throw new UnpairedSurrogateException(i, utf16Length); } throw new ArrayIndexOutOfBoundsException("Failed writing " + c + " at index " + j); } diff --git a/java/src/test/java/com/google/protobuf/DescriptorsTest.java b/java/src/test/java/com/google/protobuf/DescriptorsTest.java index edd7fc4625..30da248706 100644 --- a/java/src/test/java/com/google/protobuf/DescriptorsTest.java +++ b/java/src/test/java/com/google/protobuf/DescriptorsTest.java @@ -46,6 +46,7 @@ import com.google.protobuf.Descriptors.OneofDescriptor; import com.google.protobuf.Descriptors.ServiceDescriptor; import com.google.protobuf.test.UnittestImport; import com.google.protobuf.test.UnittestImport.ImportEnum; +import com.google.protobuf.test.UnittestImport.ImportEnumForMap; import protobuf_unittest.TestCustomOptions; import protobuf_unittest.UnittestCustomOptions; import protobuf_unittest.UnittestProto; @@ -115,7 +116,8 @@ public class DescriptorsTest extends TestCase { assertEquals(enumType, file.findEnumTypeByName("ForeignEnum")); assertNull(file.findEnumTypeByName("NoSuchType")); assertNull(file.findEnumTypeByName("protobuf_unittest.ForeignEnum")); - assertEquals(Arrays.asList(ImportEnum.getDescriptor()), + assertEquals(Arrays.asList(ImportEnum.getDescriptor(), + ImportEnumForMap.getDescriptor()), UnittestImport.getDescriptor().getEnumTypes()); for (int i = 0; i < file.getEnumTypes().size(); i++) { assertEquals(i, file.getEnumTypes().get(i).getIndex()); diff --git a/java/src/test/java/com/google/protobuf/TextFormatTest.java b/java/src/test/java/com/google/protobuf/TextFormatTest.java index 8294b8655d..1df4fad778 100644 --- a/java/src/test/java/com/google/protobuf/TextFormatTest.java +++ b/java/src/test/java/com/google/protobuf/TextFormatTest.java @@ -830,6 +830,22 @@ public class TextFormatTest extends TestCase { .build())); } + public void testShortDebugString_field() { + final FieldDescriptor dataField = + OneString.getDescriptor().findFieldByName("data"); + assertEquals( + "data: \"test data\"", + TextFormat.shortDebugString(dataField, "test data")); + + final FieldDescriptor optionalField = + TestAllTypes.getDescriptor().findFieldByName("optional_nested_message"); + final Object value = NestedMessage.newBuilder().setBb(42).build(); + + assertEquals( + "optional_nested_message { bb: 42 }", + TextFormat.shortDebugString(optionalField, value)); + } + public void testShortDebugString_unknown() { assertEquals("5: 1 5: 0x00000002 5: 0x0000000000000003 5: \"4\" 5 { 10: 5 }" + " 8: 1 8: 2 8: 3 15: 12379813812177893520 15: 0xabcd1234 15:" diff --git a/java/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java b/java/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java index e76b4a6752..dc987379fc 100644 --- a/java/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java +++ b/java/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java @@ -30,6 +30,8 @@ package com.google.protobuf; +import com.google.protobuf.UnittestLite.TestAllExtensionsLite; +import com.google.protobuf.UnittestLite.TestAllTypesLite; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Bar; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Foo; @@ -52,7 +54,40 @@ public class UnknownFieldSetLiteTest extends TestCase { UnknownFieldSetLite.newBuilder() .build()); } + + public void testBuilderReuse() throws IOException { + UnknownFieldSetLite.Builder builder = UnknownFieldSetLite.newBuilder(); + builder.mergeVarintField(10, 2); + builder.build(); + + try { + builder.build(); + fail(); + } catch (UnsupportedOperationException e) { + // Expected. + } + + try { + builder.mergeFieldFrom(0, CodedInputStream.newInstance(new byte[0])); + fail(); + } catch (UnsupportedOperationException e) { + // Expected. + } + + try { + builder.mergeVarintField(5, 1); + fail(); + } catch (UnsupportedOperationException e) { + // Expected. + } + } + public void testBuilderReuse_empty() { + UnknownFieldSetLite.Builder builder = UnknownFieldSetLite.newBuilder(); + builder.build(); + builder.build(); + } + public void testDefaultInstance() { UnknownFieldSetLite unknownFields = UnknownFieldSetLite.getDefaultInstance(); @@ -67,10 +102,10 @@ public class UnknownFieldSetLiteTest extends TestCase { CodedInputStream input = CodedInputStream.newInstance(foo.toByteArray()); - UnknownFieldSetLite.Builder builder = UnknownFieldSetLite.newBuilder(); - builder.mergeFieldFrom(input.readTag(), input); + UnknownFieldSetLite instance = UnknownFieldSetLite.newInstance(); + instance.mergeFieldFrom(input.readTag(), input); - assertEquals(foo.toByteString(), toByteString(builder.build())); + assertEquals(foo.toByteString(), toByteString(instance)); } public void testSerializedSize() throws IOException { @@ -80,18 +115,18 @@ public class UnknownFieldSetLiteTest extends TestCase { CodedInputStream input = CodedInputStream.newInstance(foo.toByteArray()); - UnknownFieldSetLite.Builder builder = UnknownFieldSetLite.newBuilder(); - builder.mergeFieldFrom(input.readTag(), input); + UnknownFieldSetLite instance = UnknownFieldSetLite.newInstance(); + instance.mergeFieldFrom(input.readTag(), input); - assertEquals(foo.toByteString().size(), builder.build().getSerializedSize()); + assertEquals(foo.toByteString().size(), instance.getSerializedSize()); } public void testMergeVarintField() throws IOException { - UnknownFieldSetLite.Builder builder = UnknownFieldSetLite.newBuilder(); - builder.mergeVarintField(10, 2); + UnknownFieldSetLite unknownFields = UnknownFieldSetLite.newInstance(); + unknownFields.mergeVarintField(10, 2); CodedInputStream input = - CodedInputStream.newInstance(toByteString(builder.build()).toByteArray()); + CodedInputStream.newInstance(toByteString(unknownFields).toByteArray()); int tag = input.readTag(); assertEquals(10, WireFormat.getTagFieldNumber(tag)); @@ -101,11 +136,11 @@ public class UnknownFieldSetLiteTest extends TestCase { } public void testMergeVarintField_negative() throws IOException { - UnknownFieldSetLite.Builder builder = UnknownFieldSetLite.newBuilder(); + UnknownFieldSetLite builder = UnknownFieldSetLite.newInstance(); builder.mergeVarintField(10, -6); CodedInputStream input = - CodedInputStream.newInstance(toByteString(builder.build()).toByteArray()); + CodedInputStream.newInstance(toByteString(builder).toByteArray()); int tag = input.readTag(); assertEquals(10, WireFormat.getTagFieldNumber(tag)); @@ -115,13 +150,11 @@ public class UnknownFieldSetLiteTest extends TestCase { } public void testEqualsAndHashCode() { - UnknownFieldSetLite.Builder builder1 = UnknownFieldSetLite.newBuilder(); - builder1.mergeVarintField(10, 2); - UnknownFieldSetLite unknownFields1 = builder1.build(); + UnknownFieldSetLite unknownFields1 = UnknownFieldSetLite.newInstance(); + unknownFields1.mergeVarintField(10, 2); - UnknownFieldSetLite.Builder builder2 = UnknownFieldSetLite.newBuilder(); - builder2.mergeVarintField(10, 2); - UnknownFieldSetLite unknownFields2 = builder2.build(); + UnknownFieldSetLite unknownFields2 = UnknownFieldSetLite.newInstance(); + unknownFields2.mergeVarintField(10, 2); assertEquals(unknownFields1, unknownFields2); assertEquals(unknownFields1.hashCode(), unknownFields2.hashCode()); @@ -129,12 +162,11 @@ public class UnknownFieldSetLiteTest extends TestCase { assertFalse(unknownFields1.hashCode() == UnknownFieldSetLite.getDefaultInstance().hashCode()); } - public void testConcat() throws IOException { - UnknownFieldSetLite.Builder builder = UnknownFieldSetLite.newBuilder(); - builder.mergeVarintField(10, 2); - UnknownFieldSetLite unknownFields = builder.build(); - - unknownFields = UnknownFieldSetLite.concat(unknownFields, unknownFields); + public void testMutableCopyOf() throws IOException { + UnknownFieldSetLite unknownFields = UnknownFieldSetLite.newInstance(); + unknownFields.mergeVarintField(10, 2); + unknownFields = UnknownFieldSetLite.mutableCopyOf(unknownFields, unknownFields); + unknownFields.checkMutable(); CodedInputStream input = CodedInputStream.newInstance(toByteString(unknownFields).toByteArray()); @@ -151,53 +183,15 @@ public class UnknownFieldSetLiteTest extends TestCase { assertTrue(input.isAtEnd()); } - public void testConcat_empty() { - UnknownFieldSetLite unknownFields = UnknownFieldSetLite.concat( + public void testMutableCopyOf_empty() { + UnknownFieldSetLite unknownFields = UnknownFieldSetLite.mutableCopyOf( UnknownFieldSetLite.getDefaultInstance(), UnknownFieldSetLite.getDefaultInstance()); + unknownFields.checkMutable(); assertEquals(0, unknownFields.getSerializedSize()); assertEquals(ByteString.EMPTY, toByteString(unknownFields)); } - public void testBuilderReuse() throws IOException { - UnknownFieldSetLite.Builder builder = UnknownFieldSetLite.newBuilder(); - builder.mergeVarintField(10, 2); - builder.build(); - - try { - builder.build(); - fail(); - } catch (IllegalStateException e) { - // Expected. - } - - try { - builder.mergeFieldFrom(0, CodedInputStream.newInstance(new byte[0])); - fail(); - } catch (IllegalStateException e) { - // Expected. - } - - try { - builder.mergeVarintField(5, 1); - fail(); - } catch (IllegalStateException e) { - // Expected. - } - } - - public void testBuilderReuse_empty() { - UnknownFieldSetLite.Builder builder = UnknownFieldSetLite.newBuilder(); - builder.build(); - - try { - builder.build(); - fail(); - } catch (IllegalStateException e) { - // Expected. - } - } - public void testRoundTrips() throws InvalidProtocolBufferException { Foo foo = Foo.newBuilder() .setValue(1) @@ -301,6 +295,64 @@ public class UnknownFieldSetLiteTest extends TestCase { // Expected. } } + + public void testMakeImmutable() throws Exception { + UnknownFieldSetLite unknownFields = UnknownFieldSetLite.newInstance(); + unknownFields.makeImmutable(); + + try { + unknownFields.mergeVarintField(1, 1); + fail(); + } catch (UnsupportedOperationException expected) {} + + try { + unknownFields.mergeLengthDelimitedField(2, ByteString.copyFromUtf8("hello")); + fail(); + } catch (UnsupportedOperationException expected) {} + + try { + unknownFields.mergeFieldFrom(1, CodedInputStream.newInstance(new byte[0])); + fail(); + } catch (UnsupportedOperationException expected) {} + } + + public void testEndToEnd() throws Exception { + TestAllTypesLite testAllTypes = TestAllTypesLite.getDefaultInstance(); + try { + testAllTypes.unknownFields.checkMutable(); + fail(); + } catch (UnsupportedOperationException expected) {} + + testAllTypes = TestAllTypesLite.parseFrom(new byte[0]); + try { + testAllTypes.unknownFields.checkMutable(); + fail(); + } catch (UnsupportedOperationException expected) {} + + testAllTypes = TestAllTypesLite.newBuilder().build(); + try { + testAllTypes.unknownFields.checkMutable(); + fail(); + } catch (UnsupportedOperationException expected) {} + + testAllTypes = TestAllTypesLite.newBuilder() + .setDefaultBool(true) + .build(); + try { + testAllTypes.unknownFields.checkMutable(); + fail(); + } catch (UnsupportedOperationException expected) {} + + TestAllExtensionsLite testAllExtensions = TestAllExtensionsLite.newBuilder() + .mergeFrom(TestAllExtensionsLite.newBuilder() + .setExtension(UnittestLite.optionalInt32ExtensionLite, 2) + .build().toByteArray()) + .build(); + try { + testAllExtensions.unknownFields.checkMutable(); + fail(); + } catch (UnsupportedOperationException expected) {} + } private ByteString toByteString(UnknownFieldSetLite unknownFields) { ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); diff --git a/java/src/test/java/com/google/protobuf/test_bad_identifiers.proto b/java/src/test/java/com/google/protobuf/test_bad_identifiers.proto index dc082615e1..8c37c03caf 100644 --- a/java/src/test/java/com/google/protobuf/test_bad_identifiers.proto +++ b/java/src/test/java/com/google/protobuf/test_bad_identifiers.proto @@ -140,11 +140,11 @@ message TestConflictingFieldNames { optional bytes bytes_field_count = 14; optional TestMessage message_field_count = 15; - repeated int32 Int32Field = 21; - repeated TestEnum EnumField = 22; - repeated string StringField = 23; - repeated bytes BytesField = 24; - repeated TestMessage MessageField = 25; + repeated int32 Int32Field = 21; // NO_PROTO3 + repeated TestEnum EnumField = 22; // NO_PROTO3 + repeated string StringField = 23; // NO_PROTO3 + repeated bytes BytesField = 24; // NO_PROTO3 + repeated TestMessage MessageField = 25; // NO_PROTO3 // This field conflicts with "int32_field" as they both generate // the method getInt32FieldList(). diff --git a/java/util/pom.xml b/java/util/pom.xml index 44a5662d74..9416f380d3 100644 --- a/java/util/pom.xml +++ b/java/util/pom.xml @@ -10,7 +10,7 @@ com.google.protobuf protobuf-java-util - 3.0.0-beta-1 + 3.0.0-alpha-4-pre bundle Protocol Buffer Java API @@ -36,7 +36,7 @@ com.google.protobuf protobuf-java - 3.0.0-beta-1 + 3.0.0-alpha-4-pre compile @@ -123,7 +123,7 @@ https://developers.google.com/protocol-buffers/ com.google.protobuf.util - com.google.protobuf.util;version=3.0.0-beta-1 + com.google.protobuf.util;version=3.0.0-alpha-3 diff --git a/python/google/protobuf/descriptor.py b/python/google/protobuf/descriptor.py index 95b703fceb..2bf3653270 100755 --- a/python/google/protobuf/descriptor.py +++ b/python/google/protobuf/descriptor.py @@ -28,8 +28,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Copyright 2007 Google Inc. All Rights Reserved. - """Descriptors essentially contain exactly the information found in a .proto file, in types that make this information accessible in Python. """ @@ -40,7 +38,6 @@ import six from google.protobuf.internal import api_implementation - _USE_C_DESCRIPTORS = False if api_implementation.Type() == 'cpp': # Used by MakeDescriptor in cpp mode @@ -221,6 +218,9 @@ class Descriptor(_NestedDescriptorBase): fields_by_name: (dict str -> FieldDescriptor) Same FieldDescriptor objects as in |fields|, but indexed by "name" attribute in each FieldDescriptor. + fields_by_camelcase_name: (dict str -> FieldDescriptor) Same + FieldDescriptor objects as in |fields|, but indexed by + "camelcase_name" attribute in each FieldDescriptor. nested_types: (list of Descriptors) Descriptor references for all protocol message types nested within this one. @@ -292,6 +292,7 @@ class Descriptor(_NestedDescriptorBase): field.containing_type = self self.fields_by_number = dict((f.number, f) for f in fields) self.fields_by_name = dict((f.name, f) for f in fields) + self._fields_by_camelcase_name = None self.nested_types = nested_types for nested_type in nested_types: @@ -317,6 +318,13 @@ class Descriptor(_NestedDescriptorBase): oneof.containing_type = self self.syntax = syntax or "proto2" + @property + def fields_by_camelcase_name(self): + if self._fields_by_camelcase_name is None: + self._fields_by_camelcase_name = dict( + (f.camelcase_name, f) for f in self.fields) + return self._fields_by_camelcase_name + def EnumValueName(self, enum, value): """Returns the string name of an enum value. @@ -365,6 +373,7 @@ class FieldDescriptor(DescriptorBase): name: (str) Name of this field, exactly as it appears in .proto. full_name: (str) Name of this field, including containing scope. This is particularly relevant for extensions. + camelcase_name: (str) Camelcase name of this field. index: (int) Dense, 0-indexed index giving the order that this field textually appears within its message in the .proto file. number: (int) Tag number declared for this field in the .proto file. @@ -509,6 +518,7 @@ class FieldDescriptor(DescriptorBase): super(FieldDescriptor, self).__init__(options, 'FieldOptions') self.name = name self.full_name = full_name + self._camelcase_name = None self.index = index self.number = number self.type = type @@ -530,6 +540,12 @@ class FieldDescriptor(DescriptorBase): else: self._cdescriptor = None + @property + def camelcase_name(self): + if self._camelcase_name is None: + self._camelcase_name = _ToCamelCase(self.name) + return self._camelcase_name + @staticmethod def ProtoTypeToCppProtoType(proto_type): """Converts from a Python proto type to a C++ Proto Type. @@ -822,6 +838,27 @@ def _ParseOptions(message, string): return message +def _ToCamelCase(name): + """Converts name to camel-case and returns it.""" + capitalize_next = False + result = [] + + for c in name: + if c == '_': + if result: + capitalize_next = True + elif capitalize_next: + result.append(c.upper()) + capitalize_next = False + else: + result += c + + # Lower-case the first letter. + if result and result[0].isupper(): + result[0] = result[0].lower() + return ''.join(result) + + def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True, syntax=None): """Make a protobuf Descriptor given a DescriptorProto protobuf. diff --git a/python/google/protobuf/internal/decoder.py b/python/google/protobuf/internal/decoder.py index 4fd7a86493..31869e4575 100755 --- a/python/google/protobuf/internal/decoder.py +++ b/python/google/protobuf/internal/decoder.py @@ -28,8 +28,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Copyright 2009 Google Inc. All Rights Reserved. - """Code for decoding protocol buffer primitives. This code is very similar to encoder.py -- read the docs for that module first. diff --git a/python/google/protobuf/internal/descriptor_database_test.py b/python/google/protobuf/internal/descriptor_database_test.py index 1baff7d1d1..3241cb7281 100644 --- a/python/google/protobuf/internal/descriptor_database_test.py +++ b/python/google/protobuf/internal/descriptor_database_test.py @@ -34,10 +34,7 @@ __author__ = 'matthewtoia@google.com (Matt Toia)' -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest from google.protobuf import descriptor_pb2 from google.protobuf.internal import factory_test2_pb2 from google.protobuf import descriptor_database diff --git a/python/google/protobuf/internal/descriptor_pool_test.py b/python/google/protobuf/internal/descriptor_pool_test.py index 2a482fbabc..6bbc8233b4 100644 --- a/python/google/protobuf/internal/descriptor_pool_test.py +++ b/python/google/protobuf/internal/descriptor_pool_test.py @@ -35,11 +35,8 @@ __author__ = 'matthewtoia@google.com (Matt Toia)' import os -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest from google.protobuf import unittest_pb2 from google.protobuf import descriptor_pb2 from google.protobuf.internal import api_implementation @@ -47,6 +44,7 @@ from google.protobuf.internal import descriptor_pool_test1_pb2 from google.protobuf.internal import descriptor_pool_test2_pb2 from google.protobuf.internal import factory_test1_pb2 from google.protobuf.internal import factory_test2_pb2 +from google.protobuf.internal import test_util from google.protobuf import descriptor from google.protobuf import descriptor_database from google.protobuf import descriptor_pool diff --git a/python/google/protobuf/internal/descriptor_test.py b/python/google/protobuf/internal/descriptor_test.py index 34843a6140..f94f9f14ec 100755 --- a/python/google/protobuf/internal/descriptor_test.py +++ b/python/google/protobuf/internal/descriptor_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # # Protocol Buffers - Google's data interchange format # Copyright 2008 Google Inc. All rights reserved. @@ -36,20 +36,17 @@ __author__ = 'robinson@google.com (Will Robinson)' import sys +import unittest from google.protobuf import unittest_custom_options_pb2 from google.protobuf import unittest_import_pb2 from google.protobuf import unittest_pb2 from google.protobuf import descriptor_pb2 from google.protobuf.internal import api_implementation +from google.protobuf.internal import test_util from google.protobuf import descriptor from google.protobuf import symbol_database from google.protobuf import text_format -try: - import unittest2 as unittest -except ImportError: - import unittest - TEST_EMPTY_MESSAGE_DESCRIPTOR_ASCII = """ name: 'TestEmptyMessage' @@ -394,7 +391,7 @@ class DescriptorTest(unittest.TestCase): self.assertEqual(self.my_file.name, 'some/filename/some.proto') self.assertEqual(self.my_file.package, 'protobuf_unittest') - @unittest.skipIf( + @test_util.skipIf( api_implementation.Type() != 'cpp' or api_implementation.Version() != 2, 'Immutability of descriptors is only enforced in v2 implementation') def testImmutableCppDescriptor(self): @@ -425,10 +422,12 @@ class GeneratedDescriptorTest(unittest.TestCase): self.CheckDescriptorSequence(message_descriptor.fields) self.CheckDescriptorMapping(message_descriptor.fields_by_name) self.CheckDescriptorMapping(message_descriptor.fields_by_number) + self.CheckDescriptorMapping(message_descriptor.fields_by_camelcase_name) def CheckFieldDescriptor(self, field_descriptor): # Basic properties self.assertEqual(field_descriptor.name, 'optional_int32') + self.assertEqual(field_descriptor.camelcase_name, 'optionalInt32') self.assertEqual(field_descriptor.full_name, 'protobuf_unittest.TestAllTypes.optional_int32') self.assertEqual(field_descriptor.containing_type.name, 'TestAllTypes') @@ -437,6 +436,10 @@ class GeneratedDescriptorTest(unittest.TestCase): self.assertEqual( field_descriptor.containing_type.fields_by_name['optional_int32'], field_descriptor) + self.assertEqual( + field_descriptor.containing_type.fields_by_camelcase_name[ + 'optionalInt32'], + field_descriptor) self.assertIn(field_descriptor, [field_descriptor]) self.assertIn(field_descriptor, {field_descriptor: None}) @@ -481,6 +484,9 @@ class GeneratedDescriptorTest(unittest.TestCase): self.CheckMessageDescriptor(message_descriptor) field_descriptor = message_descriptor.fields_by_name['optional_int32'] self.CheckFieldDescriptor(field_descriptor) + field_descriptor = message_descriptor.fields_by_camelcase_name[ + 'optionalInt32'] + self.CheckFieldDescriptor(field_descriptor) def testCppDescriptorContainer(self): # Check that the collection is still valid even if the parent disappeared. @@ -779,5 +785,20 @@ class MakeDescriptorTest(unittest.TestCase): self.assertEqual(101, options.Extensions[unittest_custom_options_pb2.msgopt].i) + def testCamelcaseName(self): + descriptor_proto = descriptor_pb2.DescriptorProto() + descriptor_proto.name = 'Bar' + names = ['foo_foo', 'FooBar', 'fooBaz', 'fooFoo', 'foobar'] + camelcase_names = ['fooFoo', 'fooBar', 'fooBaz', 'fooFoo', 'foobar'] + for index in range(len(names)): + field = descriptor_proto.field.add() + field.number = index + 1 + field.name = names[index] + result = descriptor.MakeDescriptor(descriptor_proto) + for index in range(len(camelcase_names)): + self.assertEqual(result.fields[index].camelcase_name, + camelcase_names[index]) + + if __name__ == '__main__': unittest.main() diff --git a/python/google/protobuf/internal/encoder.py b/python/google/protobuf/internal/encoder.py index d72cd29d03..48ef2df31c 100755 --- a/python/google/protobuf/internal/encoder.py +++ b/python/google/protobuf/internal/encoder.py @@ -28,8 +28,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Copyright 2009 Google Inc. All Rights Reserved. - """Code for encoding protocol message primitives. Contains the logic for encoding every logical protocol field type diff --git a/python/google/protobuf/internal/generator_test.py b/python/google/protobuf/internal/generator_test.py index 9956da59b4..7fcb137793 100755 --- a/python/google/protobuf/internal/generator_test.py +++ b/python/google/protobuf/internal/generator_test.py @@ -41,10 +41,7 @@ further ensures that we can use Python protocol message objects as we expect. __author__ = 'robinson@google.com (Will Robinson)' -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest from google.protobuf.internal import test_bad_identifiers_pb2 from google.protobuf import unittest_custom_options_pb2 from google.protobuf import unittest_import_pb2 diff --git a/python/google/protobuf/internal/json_format_test.py b/python/google/protobuf/internal/json_format_test.py new file mode 100644 index 0000000000..6d0071be33 --- /dev/null +++ b/python/google/protobuf/internal/json_format_test.py @@ -0,0 +1,522 @@ +#! /usr/bin/env python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Test for google.protobuf.json_format.""" + +__author__ = 'jieluo@google.com (Jie Luo)' + +import json +import math +import sys + +import unittest +from google.protobuf import json_format +from google.protobuf.util import json_format_proto3_pb2 + + +class JsonFormatBase(unittest.TestCase): + + def FillAllFields(self, message): + message.int32_value = 20 + message.int64_value = -20 + message.uint32_value = 3120987654 + message.uint64_value = 12345678900 + message.float_value = float('-inf') + message.double_value = 3.1415 + message.bool_value = True + message.string_value = 'foo' + message.bytes_value = b'bar' + message.message_value.value = 10 + message.enum_value = json_format_proto3_pb2.BAR + # Repeated + message.repeated_int32_value.append(0x7FFFFFFF) + message.repeated_int32_value.append(-2147483648) + message.repeated_int64_value.append(9007199254740992) + message.repeated_int64_value.append(-9007199254740992) + message.repeated_uint32_value.append(0xFFFFFFF) + message.repeated_uint32_value.append(0x7FFFFFF) + message.repeated_uint64_value.append(9007199254740992) + message.repeated_uint64_value.append(9007199254740991) + message.repeated_float_value.append(0) + + message.repeated_double_value.append(1E-15) + message.repeated_double_value.append(float('inf')) + message.repeated_bool_value.append(True) + message.repeated_bool_value.append(False) + message.repeated_string_value.append('Few symbols!#$,;') + message.repeated_string_value.append('bar') + message.repeated_bytes_value.append(b'foo') + message.repeated_bytes_value.append(b'bar') + message.repeated_message_value.add().value = 10 + message.repeated_message_value.add().value = 11 + message.repeated_enum_value.append(json_format_proto3_pb2.FOO) + message.repeated_enum_value.append(json_format_proto3_pb2.BAR) + self.message = message + + def CheckParseBack(self, message, parsed_message): + json_format.Parse(json_format.MessageToJson(message), + parsed_message) + self.assertEqual(message, parsed_message) + + def CheckError(self, text, error_message): + message = json_format_proto3_pb2.TestMessage() + self.assertRaisesRegexp( + json_format.ParseError, + error_message, + json_format.Parse, text, message) + + +class JsonFormatTest(JsonFormatBase): + + def testEmptyMessageToJson(self): + message = json_format_proto3_pb2.TestMessage() + self.assertEqual(json_format.MessageToJson(message), + '{}') + parsed_message = json_format_proto3_pb2.TestMessage() + self.CheckParseBack(message, parsed_message) + + def testPartialMessageToJson(self): + message = json_format_proto3_pb2.TestMessage( + string_value='test', + repeated_int32_value=[89, 4]) + self.assertEqual(json.loads(json_format.MessageToJson(message)), + json.loads('{"stringValue": "test", ' + '"repeatedInt32Value": [89, 4]}')) + parsed_message = json_format_proto3_pb2.TestMessage() + self.CheckParseBack(message, parsed_message) + + def testAllFieldsToJson(self): + message = json_format_proto3_pb2.TestMessage() + text = ('{"int32Value": 20, ' + '"int64Value": "-20", ' + '"uint32Value": 3120987654,' + '"uint64Value": "12345678900",' + '"floatValue": "-Infinity",' + '"doubleValue": 3.1415,' + '"boolValue": true,' + '"stringValue": "foo",' + '"bytesValue": "YmFy",' + '"messageValue": {"value": 10},' + '"enumValue": "BAR",' + '"repeatedInt32Value": [2147483647, -2147483648],' + '"repeatedInt64Value": ["9007199254740992", "-9007199254740992"],' + '"repeatedUint32Value": [268435455, 134217727],' + '"repeatedUint64Value": ["9007199254740992", "9007199254740991"],' + '"repeatedFloatValue": [0],' + '"repeatedDoubleValue": [1e-15, "Infinity"],' + '"repeatedBoolValue": [true, false],' + '"repeatedStringValue": ["Few symbols!#$,;", "bar"],' + '"repeatedBytesValue": ["Zm9v", "YmFy"],' + '"repeatedMessageValue": [{"value": 10}, {"value": 11}],' + '"repeatedEnumValue": ["FOO", "BAR"]' + '}') + self.FillAllFields(message) + self.assertEqual( + json.loads(json_format.MessageToJson(message)), + json.loads(text)) + parsed_message = json_format_proto3_pb2.TestMessage() + json_format.Parse(text, parsed_message) + self.assertEqual(message, parsed_message) + + def testJsonEscapeString(self): + message = json_format_proto3_pb2.TestMessage() + if sys.version_info[0] < 3: + message.string_value = '&\n<\"\r>\b\t\f\\\001/\xe2\x80\xa8\xe2\x80\xa9' + else: + message.string_value = '&\n<\"\r>\b\t\f\\\001/' + message.string_value += (b'\xe2\x80\xa8\xe2\x80\xa9').decode('utf-8') + self.assertEqual( + json_format.MessageToJson(message), + '{\n "stringValue": ' + '"&\\n<\\\"\\r>\\b\\t\\f\\\\\\u0001/\\u2028\\u2029"\n}') + parsed_message = json_format_proto3_pb2.TestMessage() + self.CheckParseBack(message, parsed_message) + text = u'{"int32Value": "\u0031"}' + json_format.Parse(text, message) + self.assertEqual(message.int32_value, 1) + + def testAlwaysSeriliaze(self): + message = json_format_proto3_pb2.TestMessage( + string_value='foo') + self.assertEqual( + json.loads(json_format.MessageToJson(message, True)), + json.loads('{' + '"repeatedStringValue": [],' + '"stringValue": "foo",' + '"repeatedBoolValue": [],' + '"repeatedUint32Value": [],' + '"repeatedInt32Value": [],' + '"enumValue": "FOO",' + '"int32Value": 0,' + '"floatValue": 0,' + '"int64Value": "0",' + '"uint32Value": 0,' + '"repeatedBytesValue": [],' + '"repeatedUint64Value": [],' + '"repeatedDoubleValue": [],' + '"bytesValue": "",' + '"boolValue": false,' + '"repeatedEnumValue": [],' + '"uint64Value": "0",' + '"doubleValue": 0,' + '"repeatedFloatValue": [],' + '"repeatedInt64Value": [],' + '"repeatedMessageValue": []}')) + parsed_message = json_format_proto3_pb2.TestMessage() + self.CheckParseBack(message, parsed_message) + + def testMapFields(self): + message = json_format_proto3_pb2.TestMap() + message.bool_map[True] = 1 + message.bool_map[False] = 2 + message.int32_map[1] = 2 + message.int32_map[2] = 3 + message.int64_map[1] = 2 + message.int64_map[2] = 3 + message.uint32_map[1] = 2 + message.uint32_map[2] = 3 + message.uint64_map[1] = 2 + message.uint64_map[2] = 3 + message.string_map['1'] = 2 + message.string_map['null'] = 3 + self.assertEqual( + json.loads(json_format.MessageToJson(message, True)), + json.loads('{' + '"boolMap": {"false": 2, "true": 1},' + '"int32Map": {"1": 2, "2": 3},' + '"int64Map": {"1": 2, "2": 3},' + '"uint32Map": {"1": 2, "2": 3},' + '"uint64Map": {"1": 2, "2": 3},' + '"stringMap": {"1": 2, "null": 3}' + '}')) + parsed_message = json_format_proto3_pb2.TestMap() + self.CheckParseBack(message, parsed_message) + + def testOneofFields(self): + message = json_format_proto3_pb2.TestOneof() + # Always print does not affect oneof fields. + self.assertEqual( + json_format.MessageToJson(message, True), + '{}') + message.oneof_int32_value = 0 + self.assertEqual( + json_format.MessageToJson(message, True), + '{\n' + ' "oneofInt32Value": 0\n' + '}') + parsed_message = json_format_proto3_pb2.TestOneof() + self.CheckParseBack(message, parsed_message) + + def testTimestampMessage(self): + message = json_format_proto3_pb2.TestTimestamp() + message.value.seconds = 0 + message.value.nanos = 0 + message.repeated_value.add().seconds = 20 + message.repeated_value[0].nanos = 1 + message.repeated_value.add().seconds = 0 + message.repeated_value[1].nanos = 10000 + message.repeated_value.add().seconds = 100000000 + message.repeated_value[2].nanos = 0 + # Maximum time + message.repeated_value.add().seconds = 253402300799 + message.repeated_value[3].nanos = 999999999 + # Minimum time + message.repeated_value.add().seconds = -62135596800 + message.repeated_value[4].nanos = 0 + self.assertEqual( + json.loads(json_format.MessageToJson(message, True)), + json.loads('{' + '"value": "1970-01-01T00:00:00Z",' + '"repeatedValue": [' + ' "1970-01-01T00:00:20.000000001Z",' + ' "1970-01-01T00:00:00.000010Z",' + ' "1973-03-03T09:46:40Z",' + ' "9999-12-31T23:59:59.999999999Z",' + ' "0001-01-01T00:00:00Z"' + ']' + '}')) + parsed_message = json_format_proto3_pb2.TestTimestamp() + self.CheckParseBack(message, parsed_message) + text = (r'{"value": "1972-01-01T01:00:00.01+08:00",' + r'"repeatedValue":[' + r' "1972-01-01T01:00:00.01+08:30",' + r' "1972-01-01T01:00:00.01-01:23"]}') + json_format.Parse(text, parsed_message) + self.assertEqual(parsed_message.value.seconds, 63104400) + self.assertEqual(parsed_message.value.nanos, 10000000) + self.assertEqual(parsed_message.repeated_value[0].seconds, 63106200) + self.assertEqual(parsed_message.repeated_value[1].seconds, 63070620) + + def testDurationMessage(self): + message = json_format_proto3_pb2.TestDuration() + message.value.seconds = 1 + message.repeated_value.add().seconds = 0 + message.repeated_value[0].nanos = 10 + message.repeated_value.add().seconds = -1 + message.repeated_value[1].nanos = -1000 + message.repeated_value.add().seconds = 10 + message.repeated_value[2].nanos = 11000000 + message.repeated_value.add().seconds = -315576000000 + message.repeated_value.add().seconds = 315576000000 + self.assertEqual( + json.loads(json_format.MessageToJson(message, True)), + json.loads('{' + '"value": "1s",' + '"repeatedValue": [' + ' "0.000000010s",' + ' "-1.000001s",' + ' "10.011s",' + ' "-315576000000s",' + ' "315576000000s"' + ']' + '}')) + parsed_message = json_format_proto3_pb2.TestDuration() + self.CheckParseBack(message, parsed_message) + + def testFieldMaskMessage(self): + message = json_format_proto3_pb2.TestFieldMask() + message.value.paths.append('foo.bar') + message.value.paths.append('bar') + self.assertEqual( + json_format.MessageToJson(message, True), + '{\n' + ' "value": "foo.bar,bar"\n' + '}') + parsed_message = json_format_proto3_pb2.TestFieldMask() + self.CheckParseBack(message, parsed_message) + + def testWrapperMessage(self): + message = json_format_proto3_pb2.TestWrapper() + message.bool_value.value = False + message.int32_value.value = 0 + message.string_value.value = '' + message.bytes_value.value = b'' + message.repeated_bool_value.add().value = True + message.repeated_bool_value.add().value = False + self.assertEqual( + json.loads(json_format.MessageToJson(message, True)), + json.loads('{\n' + ' "int32Value": 0,' + ' "boolValue": false,' + ' "stringValue": "",' + ' "bytesValue": "",' + ' "repeatedBoolValue": [true, false],' + ' "repeatedInt32Value": [],' + ' "repeatedUint32Value": [],' + ' "repeatedFloatValue": [],' + ' "repeatedDoubleValue": [],' + ' "repeatedBytesValue": [],' + ' "repeatedInt64Value": [],' + ' "repeatedUint64Value": [],' + ' "repeatedStringValue": []' + '}')) + parsed_message = json_format_proto3_pb2.TestWrapper() + self.CheckParseBack(message, parsed_message) + + def testParseNull(self): + message = json_format_proto3_pb2.TestMessage() + message.repeated_int32_value.append(1) + message.repeated_int32_value.append(2) + message.repeated_int32_value.append(3) + parsed_message = json_format_proto3_pb2.TestMessage() + self.FillAllFields(parsed_message) + json_format.Parse('{"int32Value": null, ' + '"int64Value": null, ' + '"uint32Value": null,' + '"uint64Value": null,' + '"floatValue": null,' + '"doubleValue": null,' + '"boolValue": null,' + '"stringValue": null,' + '"bytesValue": null,' + '"messageValue": null,' + '"enumValue": null,' + '"repeatedInt32Value": [1, 2, null, 3],' + '"repeatedInt64Value": null,' + '"repeatedUint32Value": null,' + '"repeatedUint64Value": null,' + '"repeatedFloatValue": null,' + '"repeatedDoubleValue": null,' + '"repeatedBoolValue": null,' + '"repeatedStringValue": null,' + '"repeatedBytesValue": null,' + '"repeatedMessageValue": null,' + '"repeatedEnumValue": null' + '}', + parsed_message) + self.assertEqual(message, parsed_message) + + def testNanFloat(self): + message = json_format_proto3_pb2.TestMessage() + message.float_value = float('nan') + text = '{\n "floatValue": "NaN"\n}' + self.assertEqual(json_format.MessageToJson(message), text) + parsed_message = json_format_proto3_pb2.TestMessage() + json_format.Parse(text, parsed_message) + self.assertTrue(math.isnan(parsed_message.float_value)) + + def testParseEmptyText(self): + self.CheckError('', + r'Failed to load JSON: (Expecting value)|(No JSON)') + + def testParseBadEnumValue(self): + self.CheckError( + '{"enumValue": 1}', + 'Enum value must be a string literal with double quotes. ' + 'Type "proto3.EnumType" has no value named 1.') + self.CheckError( + '{"enumValue": "baz"}', + 'Enum value must be a string literal with double quotes. ' + 'Type "proto3.EnumType" has no value named baz.') + + def testParseBadIdentifer(self): + self.CheckError('{int32Value: 1}', + (r'Failed to load JSON: Expecting property name enclosed ' + r'in double quotes: line 1')) + self.CheckError('{"unknownName": 1}', + 'Message type "proto3.TestMessage" has no field named ' + '"unknownName".') + + def testDuplicateField(self): + self.CheckError('{"int32Value": 1,\n"int32Value":2}', + 'Failed to load JSON: duplicate key int32Value') + + def testInvalidBoolValue(self): + self.CheckError('{"boolValue": 1}', + 'Failed to parse boolValue field: ' + 'Expected true or false without quotes.') + self.CheckError('{"boolValue": "true"}', + 'Failed to parse boolValue field: ' + 'Expected true or false without quotes.') + + def testInvalidIntegerValue(self): + message = json_format_proto3_pb2.TestMessage() + text = '{"int32Value": 0x12345}' + self.assertRaises(json_format.ParseError, + json_format.Parse, text, message) + self.CheckError('{"int32Value": 012345}', + (r'Failed to load JSON: Expecting \',\' delimiter: ' + r'line 1')) + self.CheckError('{"int32Value": 1.0}', + 'Failed to parse int32Value field: ' + 'Couldn\'t parse integer: 1.0') + self.CheckError('{"int32Value": " 1 "}', + 'Failed to parse int32Value field: ' + 'Couldn\'t parse integer: " 1 "') + self.CheckError('{"int32Value": 12345678901234567890}', + 'Failed to parse int32Value field: Value out of range: ' + '12345678901234567890') + self.CheckError('{"int32Value": 1e5}', + 'Failed to parse int32Value field: ' + 'Couldn\'t parse integer: 100000.0') + self.CheckError('{"uint32Value": -1}', + 'Failed to parse uint32Value field: Value out of range: -1') + + def testInvalidFloatValue(self): + self.CheckError('{"floatValue": "nan"}', + 'Failed to parse floatValue field: Couldn\'t ' + 'parse float "nan", use "NaN" instead') + + def testInvalidBytesValue(self): + self.CheckError('{"bytesValue": "AQI"}', + 'Failed to parse bytesValue field: Incorrect padding') + self.CheckError('{"bytesValue": "AQI*"}', + 'Failed to parse bytesValue field: Incorrect padding') + + def testInvalidMap(self): + message = json_format_proto3_pb2.TestMap() + text = '{"int32Map": {"null": 2, "2": 3}}' + self.assertRaisesRegexp( + json_format.ParseError, + 'Failed to parse int32Map field: Couldn\'t parse integer: "null"', + json_format.Parse, text, message) + text = '{"int32Map": {1: 2, "2": 3}}' + self.assertRaisesRegexp( + json_format.ParseError, + (r'Failed to load JSON: Expecting property name enclosed ' + r'in double quotes: line 1'), + json_format.Parse, text, message) + text = r'{"stringMap": {"a": 3, "\u0061": 2}}' + self.assertRaisesRegexp( + json_format.ParseError, + 'Failed to load JSON: duplicate key a', + json_format.Parse, text, message) + text = '{"boolMap": {"null": 1}}' + self.assertRaisesRegexp( + json_format.ParseError, + 'Failed to parse boolMap field: Expect "true" or "false", not null.', + json_format.Parse, text, message) + + def testInvalidTimestamp(self): + message = json_format_proto3_pb2.TestTimestamp() + text = '{"value": "10000-01-01T00:00:00.00Z"}' + self.assertRaisesRegexp( + json_format.ParseError, + 'time data \'10000-01-01T00:00:00\' does not match' + ' format \'%Y-%m-%dT%H:%M:%S\'', + json_format.Parse, text, message) + text = '{"value": "1970-01-01T00:00:00.0123456789012Z"}' + self.assertRaisesRegexp( + json_format.ParseError, + 'Failed to parse value field: Failed to parse Timestamp: ' + 'nanos 0123456789012 more than 9 fractional digits.', + json_format.Parse, text, message) + text = '{"value": "1972-01-01T01:00:00.01+08"}' + self.assertRaisesRegexp( + json_format.ParseError, + (r'Failed to parse value field: Invalid timezone offset value: \+08'), + json_format.Parse, text, message) + # Time smaller than minimum time. + text = '{"value": "0000-01-01T00:00:00Z"}' + self.assertRaisesRegexp( + json_format.ParseError, + 'Failed to parse value field: year is out of range', + json_format.Parse, text, message) + # Time bigger than maxinum time. + message.value.seconds = 253402300800 + self.assertRaisesRegexp( + json_format.SerializeToJsonError, + 'Failed to serialize value field: year is out of range', + json_format.MessageToJson, message) + + def testInvalidOneof(self): + message = json_format_proto3_pb2.TestOneof() + text = '{"oneofInt32Value": 1, "oneofStringValue": "2"}' + self.assertRaisesRegexp( + json_format.ParseError, + 'Message type "proto3.TestOneof"' + ' should not have multiple "oneof_value" oneof fields.', + json_format.Parse, text, message) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/google/protobuf/internal/message_factory_test.py b/python/google/protobuf/internal/message_factory_test.py index 0d880a75ea..9ec54fff53 100644 --- a/python/google/protobuf/internal/message_factory_test.py +++ b/python/google/protobuf/internal/message_factory_test.py @@ -34,10 +34,7 @@ __author__ = 'matthewtoia@google.com (Matt Toia)' -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest from google.protobuf import descriptor_pb2 from google.protobuf.internal import factory_test1_pb2 from google.protobuf.internal import factory_test2_pb2 @@ -45,7 +42,6 @@ from google.protobuf import descriptor_database from google.protobuf import descriptor_pool from google.protobuf import message_factory - class MessageFactoryTest(unittest.TestCase): def setUp(self): diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py index d99b89be61..604c426a88 100755 --- a/python/google/protobuf/internal/message_test.py +++ b/python/google/protobuf/internal/message_test.py @@ -43,22 +43,16 @@ abstract interface. __author__ = 'gps@google.com (Gregory P. Smith)' + import collections import copy import math import operator import pickle -import sys - import six +import sys -if six.PY3: - long = int - -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest from google.protobuf.internal import _parameterized from google.protobuf import map_unittest_pb2 from google.protobuf import unittest_pb2 @@ -68,6 +62,9 @@ from google.protobuf.internal import packed_field_test_pb2 from google.protobuf.internal import test_util from google.protobuf import message +if six.PY3: + long = int + # Python pre-2.6 does not have isinf() or isnan() functions, so we have # to provide our own. def isnan(val): @@ -442,7 +439,7 @@ class MessageTest(unittest.TestCase): message.repeated_nested_message.add().bb = 24 message.repeated_nested_message.add().bb = 10 message.repeated_nested_message.sort(key=lambda z: z.bb // 10) - self.assertEquals( + self.assertEqual( [13, 11, 10, 21, 20, 24, 33], [n.bb for n in message.repeated_nested_message]) @@ -451,7 +448,7 @@ class MessageTest(unittest.TestCase): pb = message.SerializeToString() message.Clear() message.MergeFromString(pb) - self.assertEquals( + self.assertEqual( [13, 11, 10, 21, 20, 24, 33], [n.bb for n in message.repeated_nested_message]) @@ -914,7 +911,6 @@ class MessageTest(unittest.TestCase): with self.assertRaises(pickle.PickleError) as _: pickle.dumps(m.repeated_int32, pickle.HIGHEST_PROTOCOL) - def testSortEmptyRepeatedCompositeContainer(self, message_module): """Exercise a scenario that has led to segfaults in the past. """ diff --git a/python/google/protobuf/internal/missing_enum_values.proto b/python/google/protobuf/internal/missing_enum_values.proto index 161fc5e14f..1850be5bb7 100644 --- a/python/google/protobuf/internal/missing_enum_values.proto +++ b/python/google/protobuf/internal/missing_enum_values.proto @@ -50,3 +50,7 @@ message TestMissingEnumValues { repeated NestedEnum repeated_nested_enum = 2; repeated NestedEnum packed_nested_enum = 3 [packed = true]; } + +message JustString { + required string dummy = 1; +} diff --git a/python/google/protobuf/internal/proto_builder_test.py b/python/google/protobuf/internal/proto_builder_test.py index 1eda10fbb9..e0467251aa 100644 --- a/python/google/protobuf/internal/proto_builder_test.py +++ b/python/google/protobuf/internal/proto_builder_test.py @@ -34,14 +34,10 @@ try: from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict #PY26 - -try: - import unittest2 as unittest #PY26 except ImportError: - import unittest - + from ordereddict import OrderedDict #PY26 +import collections +import unittest from google.protobuf import descriptor_pb2 from google.protobuf import descriptor_pool from google.protobuf import proto_builder diff --git a/python/google/protobuf/internal/python_message.py b/python/google/protobuf/internal/python_message.py index 4e5032a7ed..2b87f7045c 100755 --- a/python/google/protobuf/internal/python_message.py +++ b/python/google/protobuf/internal/python_message.py @@ -28,8 +28,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Copyright 2007 Google Inc. All Rights Reserved. -# # This code is meant to work on Python 2.4 and above only. # # TODO(robinson): Helpers for verbose, common checks like seeing if a diff --git a/python/google/protobuf/internal/reflection_test.py b/python/google/protobuf/internal/reflection_test.py index 266113533b..6815c238c6 100755 --- a/python/google/protobuf/internal/reflection_test.py +++ b/python/google/protobuf/internal/reflection_test.py @@ -38,14 +38,10 @@ pure-Python protocol compiler. import copy import gc import operator -import struct -try: - import unittest2 as unittest -except ImportError: - import unittest - import six +import struct +import unittest from google.protobuf import unittest_import_pb2 from google.protobuf import unittest_mset_pb2 from google.protobuf import unittest_pb2 @@ -1627,7 +1623,7 @@ class ReflectionTest(unittest.TestCase): self.assertFalse(proto.IsInitialized(errors)) self.assertEqual(errors, ['a', 'b', 'c']) - @unittest.skipIf( + @test_util.skipIf( api_implementation.Type() != 'cpp' or api_implementation.Version() != 2, 'Errors are only available from the most recent C++ implementation.') def testFileDescriptorErrors(self): @@ -1799,7 +1795,6 @@ class ReflectionTest(unittest.TestCase): # Just check the default value. self.assertEqual(57, msg.inner.value) - # Since we had so many tests for protocol buffer equality, we broke these out # into separate TestCase classes. @@ -2827,7 +2822,7 @@ class OptionsTest(unittest.TestCase): class ClassAPITest(unittest.TestCase): - @unittest.skipIf( + @test_util.skipIf( api_implementation.Type() == 'cpp' and api_implementation.Version() == 2, 'C++ implementation requires a call to MakeDescriptor()') def testMakeClassWithNestedDescriptor(self): diff --git a/python/google/protobuf/internal/service_reflection_test.py b/python/google/protobuf/internal/service_reflection_test.py index 98614b77f6..564e2d1a39 100755 --- a/python/google/protobuf/internal/service_reflection_test.py +++ b/python/google/protobuf/internal/service_reflection_test.py @@ -34,10 +34,7 @@ __author__ = 'petar@google.com (Petar Petrov)' -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest from google.protobuf import unittest_pb2 from google.protobuf import service_reflection from google.protobuf import service diff --git a/python/google/protobuf/internal/symbol_database_test.py b/python/google/protobuf/internal/symbol_database_test.py index 9744226296..7fb4b56d62 100644 --- a/python/google/protobuf/internal/symbol_database_test.py +++ b/python/google/protobuf/internal/symbol_database_test.py @@ -32,27 +32,29 @@ """Tests for google.protobuf.symbol_database.""" -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest from google.protobuf import unittest_pb2 +from google.protobuf import descriptor from google.protobuf import symbol_database - class SymbolDatabaseTest(unittest.TestCase): def _Database(self): - db = symbol_database.SymbolDatabase() - # Register representative types from unittest_pb2. - db.RegisterFileDescriptor(unittest_pb2.DESCRIPTOR) - db.RegisterMessage(unittest_pb2.TestAllTypes) - db.RegisterMessage(unittest_pb2.TestAllTypes.NestedMessage) - db.RegisterMessage(unittest_pb2.TestAllTypes.OptionalGroup) - db.RegisterMessage(unittest_pb2.TestAllTypes.RepeatedGroup) - db.RegisterEnumDescriptor(unittest_pb2.ForeignEnum.DESCRIPTOR) - db.RegisterEnumDescriptor(unittest_pb2.TestAllTypes.NestedEnum.DESCRIPTOR) - return db + # TODO(b/17734095): Remove this difference when the C++ implementation + # supports multiple databases. + if descriptor._USE_C_DESCRIPTORS: + return symbol_database.Default() + else: + db = symbol_database.SymbolDatabase() + # Register representative types from unittest_pb2. + db.RegisterFileDescriptor(unittest_pb2.DESCRIPTOR) + db.RegisterMessage(unittest_pb2.TestAllTypes) + db.RegisterMessage(unittest_pb2.TestAllTypes.NestedMessage) + db.RegisterMessage(unittest_pb2.TestAllTypes.OptionalGroup) + db.RegisterMessage(unittest_pb2.TestAllTypes.RepeatedGroup) + db.RegisterEnumDescriptor(unittest_pb2.ForeignEnum.DESCRIPTOR) + db.RegisterEnumDescriptor(unittest_pb2.TestAllTypes.NestedEnum.DESCRIPTOR) + return db def testGetPrototype(self): instance = self._Database().GetPrototype( diff --git a/python/google/protobuf/internal/test_util.py b/python/google/protobuf/internal/test_util.py index ac88fa81d8..539236b4cd 100755 --- a/python/google/protobuf/internal/test_util.py +++ b/python/google/protobuf/internal/test_util.py @@ -38,6 +38,13 @@ __author__ = 'robinson@google.com (Will Robinson)' import os.path +import sys +# PY2.6 compatible skipIf +if sys.version_info < (2, 7): + from unittest2 import skipIf +else: + from unittest import skipIf + from google.protobuf import unittest_import_pb2 from google.protobuf import unittest_pb2 from google.protobuf import descriptor_pb2 diff --git a/python/google/protobuf/internal/text_encoding_test.py b/python/google/protobuf/internal/text_encoding_test.py index 338a287b1b..9e7b9ce433 100755 --- a/python/google/protobuf/internal/text_encoding_test.py +++ b/python/google/protobuf/internal/text_encoding_test.py @@ -32,10 +32,7 @@ """Tests for google.protobuf.text_encoding.""" -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest from google.protobuf import text_encoding TEST_VALUES = [ diff --git a/python/google/protobuf/internal/text_format_test.py b/python/google/protobuf/internal/text_format_test.py index d332b77dbc..fb4addeb25 100755 --- a/python/google/protobuf/internal/text_format_test.py +++ b/python/google/protobuf/internal/text_format_test.py @@ -34,14 +34,12 @@ __author__ = 'kenton@google.com (Kenton Varda)' + import re import six import string -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest from google.protobuf.internal import _parameterized from google.protobuf import map_unittest_pb2 @@ -389,7 +387,7 @@ class TextFormatTest(TextFormatBase): # Ideally the schemas would be made more similar so these tests could pass. class OnlyWorksWithProto2RightNowTests(TextFormatBase): - def testPrintAllFieldsPointy(self, message_module): + def testPrintAllFieldsPointy(self): message = unittest_pb2.TestAllTypes() test_util.SetAllFields(message) self.CompareToGoldenFile( diff --git a/python/google/protobuf/internal/type_checkers.py b/python/google/protobuf/internal/type_checkers.py index 8fa3d8c835..f30ca6a80e 100755 --- a/python/google/protobuf/internal/type_checkers.py +++ b/python/google/protobuf/internal/type_checkers.py @@ -28,8 +28,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Copyright 2008 Google Inc. All Rights Reserved. - """Provides type checking routines. This module defines type checking utilities in the forms of dictionaries: @@ -52,6 +50,7 @@ import six if six.PY3: long = int +from google.protobuf.internal import api_implementation from google.protobuf.internal import decoder from google.protobuf.internal import encoder from google.protobuf.internal import wire_format diff --git a/python/google/protobuf/internal/unknown_fields_test.py b/python/google/protobuf/internal/unknown_fields_test.py index 011d3b55da..25b447e18b 100755 --- a/python/google/protobuf/internal/unknown_fields_test.py +++ b/python/google/protobuf/internal/unknown_fields_test.py @@ -35,11 +35,7 @@ __author__ = 'bohdank@google.com (Bohdan Koval)' -try: - import unittest2 as unittest -except ImportError: - import unittest - +import unittest from google.protobuf import unittest_mset_pb2 from google.protobuf import unittest_pb2 from google.protobuf import unittest_proto3_arena_pb2 @@ -52,7 +48,7 @@ from google.protobuf.internal import type_checkers def SkipIfCppImplementation(func): - return unittest.skipIf( + return test_util.skipIf( api_implementation.Type() == 'cpp' and api_implementation.Version() == 2, 'C++ implementation does not expose unknown fields to Python')(func) @@ -262,6 +258,19 @@ class UnknownEnumValuesTest(unittest.TestCase): decoder(value, 0, len(value), self.message, result_dict) return result_dict[field_descriptor] + def testUnknownParseMismatchEnumValue(self): + just_string = missing_enum_values_pb2.JustString() + just_string.dummy = 'blah' + + missing = missing_enum_values_pb2.TestEnumValues() + # The parse is invalid, storing the string proto into the set of + # unknown fields. + missing.ParseFromString(just_string.SerializeToString()) + + # Fetching the enum field shouldn't crash, instead returning the + # default value. + self.assertEqual(missing.optional_nested_enum, 0) + @SkipIfCppImplementation def testUnknownEnumValue(self): self.assertFalse(self.missing_message.HasField('optional_nested_enum')) diff --git a/python/google/protobuf/internal/wire_format_test.py b/python/google/protobuf/internal/wire_format_test.py index f659d18e42..78dc11670f 100755 --- a/python/google/protobuf/internal/wire_format_test.py +++ b/python/google/protobuf/internal/wire_format_test.py @@ -34,10 +34,7 @@ __author__ = 'robinson@google.com (Will Robinson)' -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest from google.protobuf import message from google.protobuf.internal import wire_format diff --git a/python/google/protobuf/json_format.py b/python/google/protobuf/json_format.py new file mode 100644 index 0000000000..09110e04dd --- /dev/null +++ b/python/google/protobuf/json_format.py @@ -0,0 +1,601 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Contains routines for printing protocol messages in JSON format.""" + +__author__ = 'jieluo@google.com (Jie Luo)' + +import base64 +from datetime import datetime +import json +import math +import re + +from google.protobuf import descriptor + +_TIMESTAMPFOMAT = '%Y-%m-%dT%H:%M:%S' +_NUMBER = re.compile(u'[0-9+-][0-9e.+-]*') +_INTEGER = re.compile(u'[0-9+-]') +_INT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_INT32, + descriptor.FieldDescriptor.CPPTYPE_UINT32, + descriptor.FieldDescriptor.CPPTYPE_INT64, + descriptor.FieldDescriptor.CPPTYPE_UINT64]) +_INT64_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_INT64, + descriptor.FieldDescriptor.CPPTYPE_UINT64]) +_FLOAT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_FLOAT, + descriptor.FieldDescriptor.CPPTYPE_DOUBLE]) +if str is bytes: + _UNICODETYPE = unicode +else: + _UNICODETYPE = str + + +class SerializeToJsonError(Exception): + """Thrown if serialization to JSON fails.""" + + +class ParseError(Exception): + """Thrown in case of parsing error.""" + + +def MessageToJson(message, including_default_value_fields=False): + """Converts protobuf message to JSON format. + + Args: + message: The protocol buffers message instance to serialize. + including_default_value_fields: If True, singular primitive fields, + repeated fields, and map fields will always be serialized. If + False, only serialize non-empty fields. Singular message fields + and oneof fields are not affected by this option. + + Returns: + A string containing the JSON formatted protocol buffer message. + """ + js = _MessageToJsonObject(message, including_default_value_fields) + return json.dumps(js, indent=2) + + +def _MessageToJsonObject(message, including_default_value_fields): + """Converts message to an object according to Proto3 JSON Specification.""" + message_descriptor = message.DESCRIPTOR + if _IsTimestampMessage(message_descriptor): + return _TimestampMessageToJsonObject(message) + if _IsDurationMessage(message_descriptor): + return _DurationMessageToJsonObject(message) + if _IsFieldMaskMessage(message_descriptor): + return _FieldMaskMessageToJsonObject(message) + if _IsWrapperMessage(message_descriptor): + return _WrapperMessageToJsonObject(message) + return _RegularMessageToJsonObject(message, including_default_value_fields) + + +def _IsMapEntry(field): + return (field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and + field.message_type.has_options and + field.message_type.GetOptions().map_entry) + + +def _RegularMessageToJsonObject(message, including_default_value_fields): + """Converts normal message according to Proto3 JSON Specification.""" + js = {} + fields = message.ListFields() + + try: + for field, value in fields: + name = field.camelcase_name + if _IsMapEntry(field): + # Convert a map field. + js_map = {} + for key in value: + js_map[key] = _ConvertFieldToJsonObject( + field.message_type.fields_by_name['value'], + value[key], including_default_value_fields) + js[name] = js_map + elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED: + # Convert a repeated field. + repeated = [] + for element in value: + repeated.append(_ConvertFieldToJsonObject( + field, element, including_default_value_fields)) + js[name] = repeated + else: + js[name] = _ConvertFieldToJsonObject( + field, value, including_default_value_fields) + + # Serialize default value if including_default_value_fields is True. + if including_default_value_fields: + message_descriptor = message.DESCRIPTOR + for field in message_descriptor.fields: + # Singular message fields and oneof fields will not be affected. + if ((field.label != descriptor.FieldDescriptor.LABEL_REPEATED and + field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE) or + field.containing_oneof): + continue + name = field.camelcase_name + if name in js: + # Skip the field which has been serailized already. + continue + if _IsMapEntry(field): + js[name] = {} + elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED: + js[name] = [] + else: + js[name] = _ConvertFieldToJsonObject(field, field.default_value) + + except ValueError as e: + raise SerializeToJsonError( + 'Failed to serialize {0} field: {1}'.format(field.name, e)) + + return js + + +def _ConvertFieldToJsonObject( + field, value, including_default_value_fields=False): + """Converts field value according to Proto3 JSON Specification.""" + if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: + return _MessageToJsonObject(value, including_default_value_fields) + elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM: + enum_value = field.enum_type.values_by_number.get(value, None) + if enum_value is not None: + return enum_value.name + else: + raise SerializeToJsonError('Enum field contains an integer value ' + 'which can not mapped to an enum value.') + elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING: + if field.type == descriptor.FieldDescriptor.TYPE_BYTES: + # Use base64 Data encoding for bytes + return base64.b64encode(value).decode('utf-8') + else: + return value + elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL: + if value: + return True + else: + return False + elif field.cpp_type in _INT64_TYPES: + return str(value) + elif field.cpp_type in _FLOAT_TYPES: + if math.isinf(value): + if value < 0.0: + return '-Infinity' + else: + return 'Infinity' + if math.isnan(value): + return 'NaN' + return value + + +def _IsTimestampMessage(message_descriptor): + return (message_descriptor.name == 'Timestamp' and + message_descriptor.file.name == 'google/protobuf/timestamp.proto') + + +def _TimestampMessageToJsonObject(message): + """Converts Timestamp message according to Proto3 JSON Specification.""" + nanos = message.nanos % 1e9 + dt = datetime.utcfromtimestamp( + message.seconds + (message.nanos - nanos) / 1e9) + result = dt.isoformat() + if (nanos % 1e9) == 0: + # If there are 0 fractional digits, the fractional + # point '.' should be omitted when serializing. + return result + 'Z' + if (nanos % 1e6) == 0: + # Serialize 3 fractional digits. + return result + '.%03dZ' % (nanos / 1e6) + if (nanos % 1e3) == 0: + # Serialize 6 fractional digits. + return result + '.%06dZ' % (nanos / 1e3) + # Serialize 9 fractional digits. + return result + '.%09dZ' % nanos + + +def _IsDurationMessage(message_descriptor): + return (message_descriptor.name == 'Duration' and + message_descriptor.file.name == 'google/protobuf/duration.proto') + + +def _DurationMessageToJsonObject(message): + """Converts Duration message according to Proto3 JSON Specification.""" + if message.seconds < 0 or message.nanos < 0: + result = '-' + seconds = - message.seconds + int((0 - message.nanos) / 1e9) + nanos = (0 - message.nanos) % 1e9 + else: + result = '' + seconds = message.seconds + int(message.nanos / 1e9) + nanos = message.nanos % 1e9 + result += '%d' % seconds + if (nanos % 1e9) == 0: + # If there are 0 fractional digits, the fractional + # point '.' should be omitted when serializing. + return result + 's' + if (nanos % 1e6) == 0: + # Serialize 3 fractional digits. + return result + '.%03ds' % (nanos / 1e6) + if (nanos % 1e3) == 0: + # Serialize 6 fractional digits. + return result + '.%06ds' % (nanos / 1e3) + # Serialize 9 fractional digits. + return result + '.%09ds' % nanos + + +def _IsFieldMaskMessage(message_descriptor): + return (message_descriptor.name == 'FieldMask' and + message_descriptor.file.name == 'google/protobuf/field_mask.proto') + + +def _FieldMaskMessageToJsonObject(message): + """Converts FieldMask message according to Proto3 JSON Specification.""" + result = '' + first = True + for path in message.paths: + if not first: + result += ',' + result += path + first = False + return result + + +def _IsWrapperMessage(message_descriptor): + return message_descriptor.file.name == 'google/protobuf/wrappers.proto' + + +def _WrapperMessageToJsonObject(message): + return _ConvertFieldToJsonObject( + message.DESCRIPTOR.fields_by_name['value'], message.value) + + +def _DuplicateChecker(js): + result = {} + for name, value in js: + if name in result: + raise ParseError('Failed to load JSON: duplicate key ' + name) + result[name] = value + return result + + +def Parse(text, message): + """Parses a JSON representation of a protocol message into a message. + + Args: + text: Message JSON representation. + message: A protocol beffer message to merge into. + + Returns: + The same message passed as argument. + + Raises:: + ParseError: On JSON parsing problems. + """ + if not isinstance(text, _UNICODETYPE): text = text.decode('utf-8') + try: + js = json.loads(text, object_pairs_hook=_DuplicateChecker) + except ValueError as e: + raise ParseError('Failed to load JSON: ' + str(e)) + _ConvertFieldValuePair(js, message) + return message + + +def _ConvertFieldValuePair(js, message): + """Convert field value pairs into regular message. + + Args: + js: A JSON object to convert the field value pairs. + message: A regular protocol message to record the data. + + Raises: + ParseError: In case of problems converting. + """ + names = [] + message_descriptor = message.DESCRIPTOR + for name in js: + try: + field = message_descriptor.fields_by_camelcase_name.get(name, None) + if not field: + raise ParseError( + 'Message type "{0}" has no field named "{1}".'.format( + message_descriptor.full_name, name)) + if name in names: + raise ParseError( + 'Message type "{0}" should not have multiple "{1}" fields.'.format( + message.DESCRIPTOR.full_name, name)) + names.append(name) + # Check no other oneof field is parsed. + if field.containing_oneof is not None: + oneof_name = field.containing_oneof.name + if oneof_name in names: + raise ParseError('Message type "{0}" should not have multiple "{1}" ' + 'oneof fields.'.format( + message.DESCRIPTOR.full_name, oneof_name)) + names.append(oneof_name) + + value = js[name] + if value is None: + message.ClearField(field.name) + continue + + # Parse field value. + if _IsMapEntry(field): + message.ClearField(field.name) + _ConvertMapFieldValue(value, message, field) + elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED: + message.ClearField(field.name) + if not isinstance(value, list): + raise ParseError('repeated field {0} must be in [] which is ' + '{1}'.format(name, value)) + for item in value: + if item is None: + continue + if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: + sub_message = getattr(message, field.name).add() + _ConvertMessage(item, sub_message) + else: + getattr(message, field.name).append( + _ConvertScalarFieldValue(item, field)) + elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: + sub_message = getattr(message, field.name) + _ConvertMessage(value, sub_message) + else: + setattr(message, field.name, _ConvertScalarFieldValue(value, field)) + except ParseError as e: + if field and field.containing_oneof is None: + raise ParseError('Failed to parse {0} field: {1}'.format(name, e)) + else: + raise ParseError(str(e)) + except ValueError as e: + raise ParseError('Failed to parse {0} field: {1}'.format(name, e)) + except TypeError as e: + raise ParseError('Failed to parse {0} field: {1}'.format(name, e)) + + +def _ConvertMessage(value, message): + """Convert a JSON object into a message. + + Args: + value: A JSON object. + message: A WKT or regular protocol message to record the data. + + Raises: + ParseError: In case of convert problems. + """ + message_descriptor = message.DESCRIPTOR + if _IsTimestampMessage(message_descriptor): + _ConvertTimestampMessage(value, message) + elif _IsDurationMessage(message_descriptor): + _ConvertDurationMessage(value, message) + elif _IsFieldMaskMessage(message_descriptor): + _ConvertFieldMaskMessage(value, message) + elif _IsWrapperMessage(message_descriptor): + _ConvertWrapperMessage(value, message) + else: + _ConvertFieldValuePair(value, message) + + +def _ConvertTimestampMessage(value, message): + """Convert a JSON representation into Timestamp message.""" + timezone_offset = value.find('Z') + if timezone_offset == -1: + timezone_offset = value.find('+') + if timezone_offset == -1: + timezone_offset = value.rfind('-') + if timezone_offset == -1: + raise ParseError( + 'Failed to parse timestamp: missing valid timezone offset.') + time_value = value[0:timezone_offset] + # Parse datetime and nanos + point_position = time_value.find('.') + if point_position == -1: + second_value = time_value + nano_value = '' + else: + second_value = time_value[:point_position] + nano_value = time_value[point_position + 1:] + date_object = datetime.strptime(second_value, _TIMESTAMPFOMAT) + seconds = (date_object - datetime(1970, 1, 1)).total_seconds() + if len(nano_value) > 9: + raise ParseError( + 'Failed to parse Timestamp: nanos {0} more than ' + '9 fractional digits.'.format(nano_value)) + if nano_value: + nanos = round(float('0.' + nano_value) * 1e9) + else: + nanos = 0 + # Parse timezone offsets + if value[timezone_offset] == 'Z': + if len(value) != timezone_offset + 1: + raise ParseError( + 'Failed to parse timestamp: invalid trailing data {0}.'.format(value)) + else: + timezone = value[timezone_offset:] + pos = timezone.find(':') + if pos == -1: + raise ParseError( + 'Invalid timezone offset value: ' + timezone) + if timezone[0] == '+': + seconds += (int(timezone[1:pos])*60+int(timezone[pos+1:]))*60 + else: + seconds -= (int(timezone[1:pos])*60+int(timezone[pos+1:]))*60 + # Set seconds and nanos + message.seconds = int(seconds) + message.nanos = int(nanos) + + +def _ConvertDurationMessage(value, message): + """Convert a JSON representation into Duration message.""" + if value[-1] != 's': + raise ParseError( + 'Duration must end with letter "s": ' + value) + try: + duration = float(value[:-1]) + except ValueError: + raise ParseError( + 'Couldn\'t parse duration: ' + value) + message.seconds = int(duration) + message.nanos = int(round((duration - message.seconds) * 1e9)) + + +def _ConvertFieldMaskMessage(value, message): + """Convert a JSON representation into FieldMask message.""" + for path in value.split(','): + message.paths.append(path) + + +def _ConvertWrapperMessage(value, message): + """Convert a JSON representation into Wrapper message.""" + field = message.DESCRIPTOR.fields_by_name['value'] + setattr(message, 'value', _ConvertScalarFieldValue(value, field)) + + +def _ConvertMapFieldValue(value, message, field): + """Convert map field value for a message map field. + + Args: + value: A JSON object to convert the map field value. + message: A protocol message to record the converted data. + field: The descriptor of the map field to be converted. + + Raises: + ParseError: In case of convert problems. + """ + if not isinstance(value, dict): + raise ParseError( + 'Map fieled {0} must be in {} which is {1}.'.format(field.name, value)) + key_field = field.message_type.fields_by_name['key'] + value_field = field.message_type.fields_by_name['value'] + for key in value: + key_value = _ConvertScalarFieldValue(key, key_field, True) + if value_field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: + _ConvertMessage(value[key], getattr(message, field.name)[key_value]) + else: + getattr(message, field.name)[key_value] = _ConvertScalarFieldValue( + value[key], value_field) + + +def _ConvertScalarFieldValue(value, field, require_quote=False): + """Convert a single scalar field value. + + Args: + value: A scalar value to convert the scalar field value. + field: The descriptor of the field to convert. + require_quote: If True, '"' is required for the field value. + + Returns: + The converted scalar field value + + Raises: + ParseError: In case of convert problems. + """ + if field.cpp_type in _INT_TYPES: + return _ConvertInteger(value) + elif field.cpp_type in _FLOAT_TYPES: + return _ConvertFloat(value) + elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL: + return _ConvertBool(value, require_quote) + elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING: + if field.type == descriptor.FieldDescriptor.TYPE_BYTES: + return base64.b64decode(value) + else: + return value + elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM: + # Convert an enum value. + enum_value = field.enum_type.values_by_name.get(value, None) + if enum_value is None: + raise ParseError( + 'Enum value must be a string literal with double quotes. ' + 'Type "{0}" has no value named {1}.'.format( + field.enum_type.full_name, value)) + return enum_value.number + + +def _ConvertInteger(value): + """Convert an integer. + + Args: + value: A scalar value to convert. + + Returns: + The integer value. + + Raises: + ParseError: If an integer couldn't be consumed. + """ + if isinstance(value, float): + raise ParseError('Couldn\'t parse integer: {0}'.format(value)) + + if isinstance(value, _UNICODETYPE) and not _INTEGER.match(value): + raise ParseError('Couldn\'t parse integer: "{0}"'.format(value)) + + return int(value) + + +def _ConvertFloat(value): + """Convert an floating point number.""" + if value == 'nan': + raise ParseError('Couldn\'t parse float "nan", use "NaN" instead') + try: + # Assume Python compatible syntax. + return float(value) + except ValueError: + # Check alternative spellings. + if value == '-Infinity': + return float('-inf') + elif value == 'Infinity': + return float('inf') + elif value == 'NaN': + return float('nan') + else: + raise ParseError('Couldn\'t parse float: {0}'.format(value)) + + +def _ConvertBool(value, require_quote): + """Convert a boolean value. + + Args: + value: A scalar value to convert. + require_quote: If True, '"' is required for the boolean value. + + Returns: + The bool parsed. + + Raises: + ParseError: If a boolean value couldn't be consumed. + """ + if require_quote: + if value == 'true': + return True + elif value == 'false': + return False + else: + raise ParseError('Expect "true" or "false", not {0}.'.format(value)) + + if not isinstance(value, bool): + raise ParseError('Expected true or false without quotes.') + return value diff --git a/python/google/protobuf/message_factory.py b/python/google/protobuf/message_factory.py index 36062a5693..9cd9c2a810 100644 --- a/python/google/protobuf/message_factory.py +++ b/python/google/protobuf/message_factory.py @@ -28,8 +28,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Copyright 2012 Google Inc. All Rights Reserved. - """Provides a factory class for generating dynamic messages. The easiest way to use this class is if you have access to the FileDescriptor diff --git a/python/google/protobuf/proto_builder.py b/python/google/protobuf/proto_builder.py index 700e3c25e5..736caed385 100644 --- a/python/google/protobuf/proto_builder.py +++ b/python/google/protobuf/proto_builder.py @@ -48,7 +48,7 @@ def _GetMessageFromFactory(factory, full_name): factory: a MessageFactory instance. full_name: str, the fully qualified name of the proto type. Returns: - a class, for the type identified by full_name. + A class, for the type identified by full_name. Raises: KeyError, if the proto is not found in the factory's descriptor pool. """ @@ -57,7 +57,7 @@ def _GetMessageFromFactory(factory, full_name): return proto_cls -def MakeSimpleProtoClass(fields, full_name, pool=None): +def MakeSimpleProtoClass(fields, full_name=None, pool=None): """Create a Protobuf class whose fields are basic types. Note: this doesn't validate field names! @@ -66,18 +66,20 @@ def MakeSimpleProtoClass(fields, full_name, pool=None): fields: dict of {name: field_type} mappings for each field in the proto. If this is an OrderedDict the order will be maintained, otherwise the fields will be sorted by name. - full_name: str, the fully-qualified name of the proto type. + full_name: optional str, the fully-qualified name of the proto type. pool: optional DescriptorPool instance. Returns: a class, the new protobuf class with a FileDescriptor. """ factory = message_factory.MessageFactory(pool=pool) - try: - proto_cls = _GetMessageFromFactory(factory, full_name) - return proto_cls - except KeyError: - # The factory's DescriptorPool doesn't know about this class yet. - pass + + if full_name is not None: + try: + proto_cls = _GetMessageFromFactory(factory, full_name) + return proto_cls + except KeyError: + # The factory's DescriptorPool doesn't know about this class yet. + pass # Get a list of (name, field_type) tuples from the fields dict. If fields was # an OrderedDict we keep the order, but otherwise we sort the field to ensure @@ -94,6 +96,25 @@ def MakeSimpleProtoClass(fields, full_name, pool=None): fields_hash.update(str(f_type).encode('utf-8')) proto_file_name = fields_hash.hexdigest() + '.proto' + # If the proto is anonymous, use the same hash to name it. + if full_name is None: + full_name = ('net.proto2.python.public.proto_builder.AnonymousProto_' + + fields_hash.hexdigest()) + try: + proto_cls = _GetMessageFromFactory(factory, full_name) + return proto_cls + except KeyError: + # The factory's DescriptorPool doesn't know about this class yet. + pass + + # This is the first time we see this proto: add a new descriptor to the pool. + factory.pool.Add( + _MakeFileDescriptorProto(proto_file_name, full_name, field_items)) + return _GetMessageFromFactory(factory, full_name) + + +def _MakeFileDescriptorProto(proto_file_name, full_name, field_items): + """Populate FileDescriptorProto for MessageFactory's DescriptorPool.""" package, name = full_name.rsplit('.', 1) file_proto = descriptor_pb2.FileDescriptorProto() file_proto.name = os.path.join(package.replace('.', '/'), proto_file_name) @@ -106,6 +127,4 @@ def MakeSimpleProtoClass(fields, full_name, pool=None): field_proto.number = f_number field_proto.label = descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL field_proto.type = f_type - - factory.pool.Add(file_proto) - return _GetMessageFromFactory(factory, full_name) + return file_proto diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc index 3806643f11..b238fd0272 100644 --- a/python/google/protobuf/pyext/descriptor.cc +++ b/python/google/protobuf/pyext/descriptor.cc @@ -223,8 +223,7 @@ static PyObject* GetOrBuildOptions(const DescriptorClass *descriptor) { options.SerializeToString(&serialized); io::CodedInputStream input( reinterpret_cast(serialized.c_str()), serialized.size()); - input.SetExtensionRegistry(pool->pool, - GetDescriptorPool()->message_factory); + input.SetExtensionRegistry(pool->pool, pool->message_factory); bool success = cmsg->message->MergePartialFromCodedStream(&input); if (!success) { PyErr_Format(PyExc_ValueError, "Error parsing Options message"); @@ -414,8 +413,14 @@ static PyObject* GetFile(PyBaseDescriptor *self, void *closure) { } static PyObject* GetConcreteClass(PyBaseDescriptor* self, void *closure) { + // Retuns the canonical class for the given descriptor. + // This is the class that was registered with the primary descriptor pool + // which contains this descriptor. + // This might not be the one you expect! For example the returned object does + // not know about extensions defined in a custom pool. PyObject* concrete_class(cdescriptor_pool::GetMessageClass( - GetDescriptorPool(), _GetDescriptor(self))); + GetDescriptorPool_FromPool(_GetDescriptor(self)->file()->pool()), + _GetDescriptor(self))); Py_XINCREF(concrete_class); return concrete_class; } @@ -424,6 +429,11 @@ static PyObject* GetFieldsByName(PyBaseDescriptor* self, void *closure) { return NewMessageFieldsByName(_GetDescriptor(self)); } +static PyObject* GetFieldsByCamelcaseName(PyBaseDescriptor* self, + void *closure) { + return NewMessageFieldsByCamelcaseName(_GetDescriptor(self)); +} + static PyObject* GetFieldsByNumber(PyBaseDescriptor* self, void *closure) { return NewMessageFieldsByNumber(_GetDescriptor(self)); } @@ -564,6 +574,8 @@ static PyGetSetDef Getters[] = { { "fields", (getter)GetFieldsSeq, NULL, "Fields sequence"}, { "fields_by_name", (getter)GetFieldsByName, NULL, "Fields by name"}, + { "fields_by_camelcase_name", (getter)GetFieldsByCamelcaseName, NULL, + "Fields by camelCase name"}, { "fields_by_number", (getter)GetFieldsByNumber, NULL, "Fields by number"}, { "nested_types", (getter)GetNestedTypesSeq, NULL, "Nested types sequence"}, { "nested_types_by_name", (getter)GetNestedTypesByName, NULL, @@ -662,6 +674,10 @@ static PyObject* GetName(PyBaseDescriptor *self, void *closure) { return PyString_FromCppString(_GetDescriptor(self)->name()); } +static PyObject* GetCamelcaseName(PyBaseDescriptor* self, void *closure) { + return PyString_FromCppString(_GetDescriptor(self)->camelcase_name()); +} + static PyObject* GetType(PyBaseDescriptor *self, void *closure) { return PyInt_FromLong(_GetDescriptor(self)->type()); } @@ -850,6 +866,7 @@ static int SetOptions(PyBaseDescriptor *self, PyObject *value, static PyGetSetDef Getters[] = { { "full_name", (getter)GetFullName, NULL, "Full name"}, { "name", (getter)GetName, NULL, "Unqualified name"}, + { "camelcase_name", (getter)GetCamelcaseName, NULL, "Camelcase name"}, { "type", (getter)GetType, NULL, "C++ Type"}, { "cpp_type", (getter)GetCppType, NULL, "C++ Type"}, { "label", (getter)GetLabel, NULL, "Label"}, @@ -1070,6 +1087,15 @@ PyObject* PyEnumDescriptor_FromDescriptor( &PyEnumDescriptor_Type, enum_descriptor, NULL); } +const EnumDescriptor* PyEnumDescriptor_AsDescriptor(PyObject* obj) { + if (!PyObject_TypeCheck(obj, &PyEnumDescriptor_Type)) { + PyErr_SetString(PyExc_TypeError, "Not an EnumDescriptor"); + return NULL; + } + return reinterpret_cast( + reinterpret_cast(obj)->descriptor); +} + namespace enumvalue_descriptor { // Unchecked accessor to the C++ pointer. @@ -1359,6 +1385,15 @@ PyObject* PyFileDescriptor_FromDescriptorWithSerializedPb( return py_descriptor; } +const FileDescriptor* PyFileDescriptor_AsDescriptor(PyObject* obj) { + if (!PyObject_TypeCheck(obj, &PyFileDescriptor_Type)) { + PyErr_SetString(PyExc_TypeError, "Not a FileDescriptor"); + return NULL; + } + return reinterpret_cast( + reinterpret_cast(obj)->descriptor); +} + namespace oneof_descriptor { // Unchecked accessor to the C++ pointer. diff --git a/python/google/protobuf/pyext/descriptor.h b/python/google/protobuf/pyext/descriptor.h index b2550406c3..eb99df182e 100644 --- a/python/google/protobuf/pyext/descriptor.h +++ b/python/google/protobuf/pyext/descriptor.h @@ -72,6 +72,8 @@ PyObject* PyFileDescriptor_FromDescriptorWithSerializedPb( // exception set. const Descriptor* PyMessageDescriptor_AsDescriptor(PyObject* obj); const FieldDescriptor* PyFieldDescriptor_AsDescriptor(PyObject* obj); +const EnumDescriptor* PyEnumDescriptor_AsDescriptor(PyObject* obj); +const FileDescriptor* PyFileDescriptor_AsDescriptor(PyObject* obj); // Returns the raw C++ pointer. const void* PyDescriptor_AsVoidPtr(PyObject* obj); diff --git a/python/google/protobuf/pyext/descriptor_containers.cc b/python/google/protobuf/pyext/descriptor_containers.cc index 92e11e3122..b20f5e4ffe 100644 --- a/python/google/protobuf/pyext/descriptor_containers.cc +++ b/python/google/protobuf/pyext/descriptor_containers.cc @@ -79,9 +79,12 @@ struct PyContainer; typedef int (*CountMethod)(PyContainer* self); typedef const void* (*GetByIndexMethod)(PyContainer* self, int index); typedef const void* (*GetByNameMethod)(PyContainer* self, const string& name); +typedef const void* (*GetByCamelcaseNameMethod)(PyContainer* self, + const string& name); typedef const void* (*GetByNumberMethod)(PyContainer* self, int index); typedef PyObject* (*NewObjectFromItemMethod)(const void* descriptor); typedef const string& (*GetItemNameMethod)(const void* descriptor); +typedef const string& (*GetItemCamelcaseNameMethod)(const void* descriptor); typedef int (*GetItemNumberMethod)(const void* descriptor); typedef int (*GetItemIndexMethod)(const void* descriptor); @@ -95,6 +98,9 @@ struct DescriptorContainerDef { // Retrieve item by name (usually a call to some 'FindByName' method). // Used by "by_name" mappings. GetByNameMethod get_by_name_fn; + // Retrieve item by camelcase name (usually a call to some + // 'FindByCamelcaseName' method). Used by "by_camelcase_name" mappings. + GetByCamelcaseNameMethod get_by_camelcase_name_fn; // Retrieve item by declared number (field tag, or enum value). // Used by "by_number" mappings. GetByNumberMethod get_by_number_fn; @@ -102,6 +108,9 @@ struct DescriptorContainerDef { NewObjectFromItemMethod new_object_from_item_fn; // Retrieve the name of an item. Used by iterators on "by_name" mappings. GetItemNameMethod get_item_name_fn; + // Retrieve the camelcase name of an item. Used by iterators on + // "by_camelcase_name" mappings. + GetItemCamelcaseNameMethod get_item_camelcase_name_fn; // Retrieve the number of an item. Used by iterators on "by_number" mappings. GetItemNumberMethod get_item_number_fn; // Retrieve the index of an item for the container type. @@ -125,6 +134,7 @@ struct PyContainer { enum ContainerKind { KIND_SEQUENCE, KIND_BYNAME, + KIND_BYCAMELCASENAME, KIND_BYNUMBER, } kind; }; @@ -172,6 +182,23 @@ static bool _GetItemByKey(PyContainer* self, PyObject* key, const void** item) { self, string(name, name_size)); return true; } + case PyContainer::KIND_BYCAMELCASENAME: + { + char* camelcase_name; + Py_ssize_t name_size; + if (PyString_AsStringAndSize(key, &camelcase_name, &name_size) < 0) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + // Not a string, cannot be in the container. + PyErr_Clear(); + *item = NULL; + return true; + } + return false; + } + *item = self->container_def->get_by_camelcase_name_fn( + self, string(camelcase_name, name_size)); + return true; + } case PyContainer::KIND_BYNUMBER: { Py_ssize_t number = PyNumber_AsSsize_t(key, NULL); @@ -203,6 +230,12 @@ static PyObject* _NewKey_ByIndex(PyContainer* self, Py_ssize_t index) { const string& name(self->container_def->get_item_name_fn(item)); return PyString_FromStringAndSize(name.c_str(), name.size()); } + case PyContainer::KIND_BYCAMELCASENAME: + { + const string& name( + self->container_def->get_item_camelcase_name_fn(item)); + return PyString_FromStringAndSize(name.c_str(), name.size()); + } case PyContainer::KIND_BYNUMBER: { int value = self->container_def->get_item_number_fn(item); @@ -276,6 +309,9 @@ static PyObject* ContainerRepr(PyContainer* self) { case PyContainer::KIND_BYNAME: kind = "mapping by name"; break; + case PyContainer::KIND_BYCAMELCASENAME: + kind = "mapping by camelCase name"; + break; case PyContainer::KIND_BYNUMBER: kind = "mapping by number"; break; @@ -731,6 +767,18 @@ static PyObject* NewMappingByName( return reinterpret_cast(self); } +static PyObject* NewMappingByCamelcaseName( + DescriptorContainerDef* container_def, const void* descriptor) { + PyContainer* self = PyObject_New(PyContainer, &DescriptorMapping_Type); + if (self == NULL) { + return NULL; + } + self->descriptor = descriptor; + self->container_def = container_def; + self->kind = PyContainer::KIND_BYCAMELCASENAME; + return reinterpret_cast(self); +} + static PyObject* NewMappingByNumber( DescriptorContainerDef* container_def, const void* descriptor) { if (container_def->get_by_number_fn == NULL || @@ -889,6 +937,11 @@ static ItemDescriptor GetByName(PyContainer* self, const string& name) { return GetDescriptor(self)->FindFieldByName(name); } +static ItemDescriptor GetByCamelcaseName(PyContainer* self, + const string& name) { + return GetDescriptor(self)->FindFieldByCamelcaseName(name); +} + static ItemDescriptor GetByNumber(PyContainer* self, int number) { return GetDescriptor(self)->FindFieldByNumber(number); } @@ -905,6 +958,10 @@ static const string& GetItemName(ItemDescriptor item) { return item->name(); } +static const string& GetItemCamelcaseName(ItemDescriptor item) { + return item->camelcase_name(); +} + static int GetItemNumber(ItemDescriptor item) { return item->number(); } @@ -918,9 +975,11 @@ static DescriptorContainerDef ContainerDef = { (CountMethod)Count, (GetByIndexMethod)GetByIndex, (GetByNameMethod)GetByName, + (GetByCamelcaseNameMethod)GetByCamelcaseName, (GetByNumberMethod)GetByNumber, (NewObjectFromItemMethod)NewObjectFromItem, (GetItemNameMethod)GetItemName, + (GetItemCamelcaseNameMethod)GetItemCamelcaseName, (GetItemNumberMethod)GetItemNumber, (GetItemIndexMethod)GetItemIndex, }; @@ -931,6 +990,11 @@ PyObject* NewMessageFieldsByName(ParentDescriptor descriptor) { return descriptor::NewMappingByName(&fields::ContainerDef, descriptor); } +PyObject* NewMessageFieldsByCamelcaseName(ParentDescriptor descriptor) { + return descriptor::NewMappingByCamelcaseName(&fields::ContainerDef, + descriptor); +} + PyObject* NewMessageFieldsByNumber(ParentDescriptor descriptor) { return descriptor::NewMappingByNumber(&fields::ContainerDef, descriptor); } @@ -972,9 +1036,11 @@ static DescriptorContainerDef ContainerDef = { (CountMethod)Count, (GetByIndexMethod)GetByIndex, (GetByNameMethod)GetByName, + (GetByCamelcaseNameMethod)NULL, (GetByNumberMethod)NULL, (NewObjectFromItemMethod)NewObjectFromItem, (GetItemNameMethod)GetItemName, + (GetItemCamelcaseNameMethod)NULL, (GetItemNumberMethod)NULL, (GetItemIndexMethod)GetItemIndex, }; @@ -1022,9 +1088,11 @@ static DescriptorContainerDef ContainerDef = { (CountMethod)Count, (GetByIndexMethod)GetByIndex, (GetByNameMethod)GetByName, + (GetByCamelcaseNameMethod)NULL, (GetByNumberMethod)NULL, (NewObjectFromItemMethod)NewObjectFromItem, (GetItemNameMethod)GetItemName, + (GetItemCamelcaseNameMethod)NULL, (GetItemNumberMethod)NULL, (GetItemIndexMethod)GetItemIndex, }; @@ -1094,9 +1162,11 @@ static DescriptorContainerDef ContainerDef = { (CountMethod)Count, (GetByIndexMethod)GetByIndex, (GetByNameMethod)GetByName, + (GetByCamelcaseNameMethod)NULL, (GetByNumberMethod)NULL, (NewObjectFromItemMethod)NewObjectFromItem, (GetItemNameMethod)GetItemName, + (GetItemCamelcaseNameMethod)NULL, (GetItemNumberMethod)NULL, (GetItemIndexMethod)NULL, }; @@ -1140,9 +1210,11 @@ static DescriptorContainerDef ContainerDef = { (CountMethod)Count, (GetByIndexMethod)GetByIndex, (GetByNameMethod)GetByName, + (GetByCamelcaseNameMethod)NULL, (GetByNumberMethod)NULL, (NewObjectFromItemMethod)NewObjectFromItem, (GetItemNameMethod)GetItemName, + (GetItemCamelcaseNameMethod)NULL, (GetItemNumberMethod)NULL, (GetItemIndexMethod)GetItemIndex, }; @@ -1190,9 +1262,11 @@ static DescriptorContainerDef ContainerDef = { (CountMethod)Count, (GetByIndexMethod)GetByIndex, (GetByNameMethod)GetByName, + (GetByCamelcaseNameMethod)NULL, (GetByNumberMethod)NULL, (NewObjectFromItemMethod)NewObjectFromItem, (GetItemNameMethod)GetItemName, + (GetItemCamelcaseNameMethod)NULL, (GetItemNumberMethod)NULL, (GetItemIndexMethod)GetItemIndex, }; @@ -1258,9 +1332,11 @@ static DescriptorContainerDef ContainerDef = { (CountMethod)Count, (GetByIndexMethod)GetByIndex, (GetByNameMethod)GetByName, + (GetByCamelcaseNameMethod)NULL, (GetByNumberMethod)GetByNumber, (NewObjectFromItemMethod)NewObjectFromItem, (GetItemNameMethod)GetItemName, + (GetItemCamelcaseNameMethod)NULL, (GetItemNumberMethod)GetItemNumber, (GetItemIndexMethod)GetItemIndex, }; @@ -1314,9 +1390,11 @@ static DescriptorContainerDef ContainerDef = { (CountMethod)Count, (GetByIndexMethod)GetByIndex, (GetByNameMethod)NULL, + (GetByCamelcaseNameMethod)NULL, (GetByNumberMethod)NULL, (NewObjectFromItemMethod)NewObjectFromItem, (GetItemNameMethod)NULL, + (GetItemCamelcaseNameMethod)NULL, (GetItemNumberMethod)NULL, (GetItemIndexMethod)GetItemIndex, }; @@ -1370,9 +1448,11 @@ static DescriptorContainerDef ContainerDef = { (CountMethod)Count, (GetByIndexMethod)GetByIndex, (GetByNameMethod)GetByName, + (GetByCamelcaseNameMethod)NULL, (GetByNumberMethod)NULL, (NewObjectFromItemMethod)NewObjectFromItem, (GetItemNameMethod)GetItemName, + (GetItemCamelcaseNameMethod)NULL, (GetItemNumberMethod)NULL, (GetItemIndexMethod)GetItemIndex, }; @@ -1416,9 +1496,11 @@ static DescriptorContainerDef ContainerDef = { (CountMethod)Count, (GetByIndexMethod)GetByIndex, (GetByNameMethod)GetByName, + (GetByCamelcaseNameMethod)NULL, (GetByNumberMethod)NULL, (NewObjectFromItemMethod)NewObjectFromItem, (GetItemNameMethod)GetItemName, + (GetItemCamelcaseNameMethod)NULL, (GetItemNumberMethod)NULL, (GetItemIndexMethod)GetItemIndex, }; @@ -1462,9 +1544,11 @@ static DescriptorContainerDef ContainerDef = { (CountMethod)Count, (GetByIndexMethod)GetByIndex, (GetByNameMethod)GetByName, + (GetByCamelcaseNameMethod)NULL, (GetByNumberMethod)NULL, (NewObjectFromItemMethod)NewObjectFromItem, (GetItemNameMethod)GetItemName, + (GetItemCamelcaseNameMethod)NULL, (GetItemNumberMethod)NULL, (GetItemIndexMethod)GetItemIndex, }; @@ -1496,9 +1580,11 @@ static DescriptorContainerDef ContainerDef = { (CountMethod)Count, (GetByIndexMethod)GetByIndex, (GetByNameMethod)NULL, + (GetByCamelcaseNameMethod)NULL, (GetByNumberMethod)NULL, (NewObjectFromItemMethod)NewObjectFromItem, (GetItemNameMethod)NULL, + (GetItemCamelcaseNameMethod)NULL, (GetItemNumberMethod)NULL, (GetItemIndexMethod)NULL, }; @@ -1530,9 +1616,11 @@ static DescriptorContainerDef ContainerDef = { (CountMethod)Count, (GetByIndexMethod)GetByIndex, (GetByNameMethod)NULL, + (GetByCamelcaseNameMethod)NULL, (GetByNumberMethod)NULL, (NewObjectFromItemMethod)NewObjectFromItem, (GetItemNameMethod)NULL, + (GetItemCamelcaseNameMethod)NULL, (GetItemNumberMethod)NULL, (GetItemIndexMethod)NULL, }; diff --git a/python/google/protobuf/pyext/descriptor_containers.h b/python/google/protobuf/pyext/descriptor_containers.h index 8fbdaff9d3..ce40747d57 100644 --- a/python/google/protobuf/pyext/descriptor_containers.h +++ b/python/google/protobuf/pyext/descriptor_containers.h @@ -54,6 +54,7 @@ bool InitDescriptorMappingTypes(); namespace message_descriptor { PyObject* NewMessageFieldsByName(const Descriptor* descriptor); +PyObject* NewMessageFieldsByCamelcaseName(const Descriptor* descriptor); PyObject* NewMessageFieldsByNumber(const Descriptor* descriptor); PyObject* NewMessageFieldsSeq(const Descriptor* descriptor); diff --git a/python/google/protobuf/pyext/descriptor_pool.cc b/python/google/protobuf/pyext/descriptor_pool.cc index 7aed651d3f..6443a7d5c9 100644 --- a/python/google/protobuf/pyext/descriptor_pool.cc +++ b/python/google/protobuf/pyext/descriptor_pool.cc @@ -108,6 +108,7 @@ static void Dealloc(PyDescriptorPool* self) { Py_DECREF(it->second); } delete self->descriptor_options; + delete self->pool; delete self->message_factory; Py_TYPE(self)->tp_free(reinterpret_cast(self)); } @@ -131,22 +132,9 @@ PyObject* FindMessageByName(PyDescriptorPool* self, PyObject* arg) { } // Add a message class to our database. -const Descriptor* RegisterMessageClass( - PyDescriptorPool* self, PyObject *message_class, PyObject* descriptor) { - ScopedPyObjectPtr full_message_name( - PyObject_GetAttrString(descriptor, "full_name")); - Py_ssize_t name_size; - char* name; - if (PyString_AsStringAndSize(full_message_name, &name, &name_size) < 0) { - return NULL; - } - const Descriptor *message_descriptor = - self->pool->FindMessageTypeByName(string(name, name_size)); - if (!message_descriptor) { - PyErr_Format(PyExc_TypeError, "Could not find C++ descriptor for '%s'", - name); - return NULL; - } +int RegisterMessageClass(PyDescriptorPool* self, + const Descriptor *message_descriptor, + PyObject *message_class) { Py_INCREF(message_class); typedef PyDescriptorPool::ClassesByMessageMap::iterator iterator; std::pair ret = self->classes_by_descriptor->insert( @@ -156,7 +144,7 @@ const Descriptor* RegisterMessageClass( Py_DECREF(ret.first->second); ret.first->second = message_class; } - return message_descriptor; + return 0; } // Retrieve the message class added to our database. @@ -260,6 +248,80 @@ PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg) { return PyOneofDescriptor_FromDescriptor(oneof_descriptor); } +PyObject* FindFileContainingSymbol(PyDescriptorPool* self, PyObject* arg) { + Py_ssize_t name_size; + char* name; + if (PyString_AsStringAndSize(arg, &name, &name_size) < 0) { + return NULL; + } + + const FileDescriptor* file_descriptor = + self->pool->FindFileContainingSymbol(string(name, name_size)); + if (file_descriptor == NULL) { + PyErr_Format(PyExc_KeyError, "Couldn't find symbol %.200s", name); + return NULL; + } + + return PyFileDescriptor_FromDescriptor(file_descriptor); +} + +// These functions should not exist -- the only valid way to create +// descriptors is to call Add() or AddSerializedFile(). +// But these AddDescriptor() functions were created in Python and some people +// call them, so we support them for now for compatibility. +// However we do check that the existing descriptor already exists in the pool, +// which appears to always be true for existing calls -- but then why do people +// call a function that will just be a no-op? +// TODO(amauryfa): Need to investigate further. + +PyObject* AddFileDescriptor(PyDescriptorPool* self, PyObject* descriptor) { + const FileDescriptor* file_descriptor = + PyFileDescriptor_AsDescriptor(descriptor); + if (!file_descriptor) { + return NULL; + } + if (file_descriptor != + self->pool->FindFileByName(file_descriptor->name())) { + PyErr_Format(PyExc_ValueError, + "The file descriptor %s does not belong to this pool", + file_descriptor->name().c_str()); + return NULL; + } + Py_RETURN_NONE; +} + +PyObject* AddDescriptor(PyDescriptorPool* self, PyObject* descriptor) { + const Descriptor* message_descriptor = + PyMessageDescriptor_AsDescriptor(descriptor); + if (!message_descriptor) { + return NULL; + } + if (message_descriptor != + self->pool->FindMessageTypeByName(message_descriptor->full_name())) { + PyErr_Format(PyExc_ValueError, + "The message descriptor %s does not belong to this pool", + message_descriptor->full_name().c_str()); + return NULL; + } + Py_RETURN_NONE; +} + +PyObject* AddEnumDescriptor(PyDescriptorPool* self, PyObject* descriptor) { + const EnumDescriptor* enum_descriptor = + PyEnumDescriptor_AsDescriptor(descriptor); + if (!enum_descriptor) { + return NULL; + } + if (enum_descriptor != + self->pool->FindEnumTypeByName(enum_descriptor->full_name())) { + PyErr_Format(PyExc_ValueError, + "The enum descriptor %s does not belong to this pool", + enum_descriptor->full_name().c_str()); + return NULL; + } + Py_RETURN_NONE; +} + // The code below loads new Descriptors from a serialized FileDescriptorProto. @@ -341,6 +403,15 @@ static PyMethodDef Methods[] = { { "AddSerializedFile", (PyCFunction)AddSerializedFile, METH_O, "Adds a serialized FileDescriptorProto to this pool." }, + // TODO(amauryfa): Understand why the Python implementation differs from + // this one, ask users to use another API and deprecate these functions. + { "AddFileDescriptor", (PyCFunction)AddFileDescriptor, METH_O, + "No-op. Add() must have been called before." }, + { "AddDescriptor", (PyCFunction)AddDescriptor, METH_O, + "No-op. Add() must have been called before." }, + { "AddEnumDescriptor", (PyCFunction)AddEnumDescriptor, METH_O, + "No-op. Add() must have been called before." }, + { "FindFileByName", (PyCFunction)FindFileByName, METH_O, "Searches for a file descriptor by its .proto name." }, { "FindMessageTypeByName", (PyCFunction)FindMessageByName, METH_O, @@ -353,6 +424,9 @@ static PyMethodDef Methods[] = { "Searches for enum type descriptor by full name." }, { "FindOneofByName", (PyCFunction)FindOneofByName, METH_O, "Searches for oneof descriptor by full name." }, + + { "FindFileContainingSymbol", (PyCFunction)FindFileContainingSymbol, METH_O, + "Gets the FileDescriptor containing the specified symbol." }, {NULL} }; @@ -420,7 +494,7 @@ bool InitDescriptorPool() { return true; } -PyDescriptorPool* GetDescriptorPool() { +PyDescriptorPool* GetDefaultDescriptorPool() { return python_generated_pool; } @@ -432,7 +506,7 @@ PyDescriptorPool* GetDescriptorPool_FromPool(const DescriptorPool* pool) { } hash_map::iterator it = descriptor_pool_map.find(pool); - if (it != descriptor_pool_map.end()) { + if (it == descriptor_pool_map.end()) { PyErr_SetString(PyExc_KeyError, "Unknown descriptor pool"); return NULL; } diff --git a/python/google/protobuf/pyext/descriptor_pool.h b/python/google/protobuf/pyext/descriptor_pool.h index 541d920bc8..eda73d385e 100644 --- a/python/google/protobuf/pyext/descriptor_pool.h +++ b/python/google/protobuf/pyext/descriptor_pool.h @@ -89,12 +89,10 @@ const Descriptor* FindMessageTypeByName(PyDescriptorPool* self, const string& name); // Registers a new Python class for the given message descriptor. -// Returns the message Descriptor. -// On error, returns NULL with a Python exception set. -const Descriptor* RegisterMessageClass( - PyDescriptorPool* self, PyObject* message_class, PyObject* descriptor); - -// The function below are also exposed as methods of the DescriptorPool type. +// On error, returns -1 with a Python exception set. +int RegisterMessageClass(PyDescriptorPool* self, + const Descriptor* message_descriptor, + PyObject* message_class); // Retrieves the Python class registered with the given message descriptor. // @@ -103,6 +101,8 @@ const Descriptor* RegisterMessageClass( PyObject* GetMessageClass(PyDescriptorPool* self, const Descriptor* message_descriptor); +// The functions below are also exposed as methods of the DescriptorPool type. + // Looks up a message by name. Returns a PyMessageDescriptor corresponding to // the field on success, or NULL on failure. // @@ -136,8 +136,9 @@ PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg); } // namespace cdescriptor_pool // Retrieve the global descriptor pool owned by the _message module. +// This is the one used by pb2.py generated modules. // Returns a *borrowed* reference. -PyDescriptorPool* GetDescriptorPool(); +PyDescriptorPool* GetDefaultDescriptorPool(); // Retrieve the python descriptor pool owning a C++ descriptor pool. // Returns a *borrowed* reference. diff --git a/python/google/protobuf/pyext/extension_dict.cc b/python/google/protobuf/pyext/extension_dict.cc index 8ebbb27c46..9c9b417822 100644 --- a/python/google/protobuf/pyext/extension_dict.cc +++ b/python/google/protobuf/pyext/extension_dict.cc @@ -123,7 +123,8 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) { if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) { if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { PyObject *message_class = cdescriptor_pool::GetMessageClass( - GetDescriptorPool(), descriptor->message_type()); + cmessage::GetDescriptorPoolForMessage(self->parent), + descriptor->message_type()); if (message_class == NULL) { return NULL; } diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index 62c7c478f5..63d531364b 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc @@ -55,6 +55,7 @@ #include #include #include +#include #include #include #include @@ -107,8 +108,18 @@ struct PyMessageMeta { // C++ descriptor of this message. const Descriptor* message_descriptor; + // Owned reference, used to keep the pointer above alive. PyObject* py_message_descriptor; + + // The Python DescriptorPool used to create the class. It is needed to resolve + // fields descriptors, including extensions fields; its C++ MessageFactory is + // used to instantiate submessages. + // This can be different from DESCRIPTOR.file.pool, in the case of a custom + // DescriptorPool which defines new extensions. + // We own the reference, because it's important to keep the descriptors and + // factory alive. + PyDescriptorPool* py_descriptor_pool; }; namespace message_meta { @@ -139,18 +150,10 @@ static bool AddFieldNumberToClass( // Finalize the creation of the Message class. -// Called from its metaclass: GeneratedProtocolMessageType.__init__(). -static int AddDescriptors(PyObject* cls, PyObject* descriptor) { - const Descriptor* message_descriptor = - cdescriptor_pool::RegisterMessageClass( - GetDescriptorPool(), cls, descriptor); - if (message_descriptor == NULL) { - return -1; - } - +static int AddDescriptors(PyObject* cls, const Descriptor* descriptor) { // If there are extension_ranges, the message is "extendable", and extension // classes will register themselves in this class. - if (message_descriptor->extension_range_count() > 0) { + if (descriptor->extension_range_count() > 0) { ScopedPyObjectPtr by_name(PyDict_New()); if (PyObject_SetAttr(cls, k_extensions_by_name, by_name) < 0) { return -1; @@ -162,8 +165,8 @@ static int AddDescriptors(PyObject* cls, PyObject* descriptor) { } // For each field set: cls._FIELD_NUMBER = - for (int i = 0; i < message_descriptor->field_count(); ++i) { - if (!AddFieldNumberToClass(cls, message_descriptor->field(i))) { + for (int i = 0; i < descriptor->field_count(); ++i) { + if (!AddFieldNumberToClass(cls, descriptor->field(i))) { return -1; } } @@ -173,8 +176,8 @@ static int AddDescriptors(PyObject* cls, PyObject* descriptor) { // The enum descriptor we get from // .enum_types_by_name[name] // which was built previously. - for (int i = 0; i < message_descriptor->enum_type_count(); ++i) { - const EnumDescriptor* enum_descriptor = message_descriptor->enum_type(i); + for (int i = 0; i < descriptor->enum_type_count(); ++i) { + const EnumDescriptor* enum_descriptor = descriptor->enum_type(i); ScopedPyObjectPtr enum_type( PyEnumDescriptor_FromDescriptor(enum_descriptor)); if (enum_type == NULL) { @@ -212,8 +215,8 @@ static int AddDescriptors(PyObject* cls, PyObject* descriptor) { // Extension descriptors come from // .extensions_by_name[name] // which was defined previously. - for (int i = 0; i < message_descriptor->extension_count(); ++i) { - const google::protobuf::FieldDescriptor* field = message_descriptor->extension(i); + for (int i = 0; i < descriptor->extension_count(); ++i) { + const google::protobuf::FieldDescriptor* field = descriptor->extension(i); ScopedPyObjectPtr extension_field(PyFieldDescriptor_FromDescriptor(field)); if (extension_field == NULL) { return -1; @@ -258,14 +261,14 @@ static PyObject* New(PyTypeObject* type, } // Check dict['DESCRIPTOR'] - PyObject* descriptor = PyDict_GetItem(dict, kDESCRIPTOR); - if (descriptor == NULL) { + PyObject* py_descriptor = PyDict_GetItem(dict, kDESCRIPTOR); + if (py_descriptor == NULL) { PyErr_SetString(PyExc_TypeError, "Message class has no DESCRIPTOR"); return NULL; } - if (!PyObject_TypeCheck(descriptor, &PyMessageDescriptor_Type)) { + if (!PyObject_TypeCheck(py_descriptor, &PyMessageDescriptor_Type)) { PyErr_Format(PyExc_TypeError, "Expected a message Descriptor, got %s", - descriptor->ob_type->tp_name); + py_descriptor->ob_type->tp_name); return NULL; } @@ -291,14 +294,28 @@ static PyObject* New(PyTypeObject* type, } // Cache the descriptor, both as Python object and as C++ pointer. - const Descriptor* message_descriptor = - PyMessageDescriptor_AsDescriptor(descriptor); - if (message_descriptor == NULL) { + const Descriptor* descriptor = + PyMessageDescriptor_AsDescriptor(py_descriptor); + if (descriptor == NULL) { + return NULL; + } + Py_INCREF(py_descriptor); + newtype->py_message_descriptor = py_descriptor; + newtype->message_descriptor = descriptor; + // TODO(amauryfa): Don't always use the canonical pool of the descriptor, + // use the MessageFactory optionally passed in the class dict. + newtype->py_descriptor_pool = GetDescriptorPool_FromPool( + descriptor->file()->pool()); + if (newtype->py_descriptor_pool == NULL) { + return NULL; + } + Py_INCREF(newtype->py_descriptor_pool); + + // Add the message to the DescriptorPool. + if (cdescriptor_pool::RegisterMessageClass(newtype->py_descriptor_pool, + descriptor, result) < 0) { return NULL; } - Py_INCREF(descriptor); - newtype->py_message_descriptor = descriptor; - newtype->message_descriptor = message_descriptor; // Continue with type initialization: add other descriptors, enum values... if (AddDescriptors(result, descriptor) < 0) { @@ -309,6 +326,7 @@ static PyObject* New(PyTypeObject* type, static void Dealloc(PyMessageMeta *self) { Py_DECREF(self->py_message_descriptor); + Py_DECREF(self->py_descriptor_pool); Py_TYPE(self)->tp_free(reinterpret_cast(self)); } @@ -381,12 +399,20 @@ PyTypeObject PyMessageMeta_Type = { message_meta::New, // tp_new }; -static const Descriptor* GetMessageDescriptor(PyTypeObject* cls) { +static PyMessageMeta* CheckMessageClass(PyTypeObject* cls) { if (!PyObject_TypeCheck(cls, &PyMessageMeta_Type)) { PyErr_Format(PyExc_TypeError, "Class %s is not a Message", cls->tp_name); return NULL; } - return reinterpret_cast(cls)->message_descriptor; + return reinterpret_cast(cls); +} + +static const Descriptor* GetMessageDescriptor(PyTypeObject* cls) { + PyMessageMeta* type = CheckMessageClass(cls); + if (type == NULL) { + return NULL; + } + return type->message_descriptor; } // Forward declarations @@ -723,6 +749,17 @@ bool CheckFieldBelongsToMessage(const FieldDescriptor* field_descriptor, namespace cmessage { +PyDescriptorPool* GetDescriptorPoolForMessage(CMessage* message) { + // No need to check the type: the type of instances of CMessage is always + // an instance of PyMessageMeta. Let's prove it with a debug-only check. + GOOGLE_DCHECK(PyObject_TypeCheck(message, &CMessage_Type)); + return reinterpret_cast(Py_TYPE(message))->py_descriptor_pool; +} + +MessageFactory* GetFactoryForMessage(CMessage* message) { + return GetDescriptorPoolForMessage(message)->message_factory; +} + static int MaybeReleaseOverlappingOneofField( CMessage* cmessage, const FieldDescriptor* field) { @@ -773,7 +810,7 @@ static Message* GetMutableMessage( return NULL; } return reflection->MutableMessage( - parent_message, parent_field, GetDescriptorPool()->message_factory); + parent_message, parent_field, GetFactoryForMessage(parent)); } struct FixupMessageReference : public ChildVisitor { @@ -814,10 +851,7 @@ int AssureWritable(CMessage* self) { // If parent is NULL but we are trying to modify a read-only message, this // is a reference to a constant default instance that needs to be replaced // with a mutable top-level message. - const Message* prototype = - GetDescriptorPool()->message_factory->GetPrototype( - self->message->GetDescriptor()); - self->message = prototype->New(); + self->message = self->message->New(); self->owner.reset(self->message); // Cascade the new owner to eventual children: even if this message is // empty, some submessages or repeated containers might exist already. @@ -1190,15 +1224,19 @@ CMessage* NewEmptyMessage(PyObject* type, const Descriptor *descriptor) { // The __new__ method of Message classes. // Creates a new C++ message and takes ownership. -static PyObject* New(PyTypeObject* type, +static PyObject* New(PyTypeObject* cls, PyObject* unused_args, PyObject* unused_kwargs) { + PyMessageMeta* type = CheckMessageClass(cls); + if (type == NULL) { + return NULL; + } // Retrieve the message descriptor and the default instance (=prototype). - const Descriptor* message_descriptor = GetMessageDescriptor(type); + const Descriptor* message_descriptor = type->message_descriptor; if (message_descriptor == NULL) { return NULL; } - const Message* default_message = - GetDescriptorPool()->message_factory->GetPrototype(message_descriptor); + const Message* default_message = type->py_descriptor_pool->message_factory + ->GetPrototype(message_descriptor); if (default_message == NULL) { PyErr_SetString(PyExc_TypeError, message_descriptor->full_name().c_str()); return NULL; @@ -1528,7 +1566,7 @@ int SetOwner(CMessage* self, const shared_ptr& new_owner) { Message* ReleaseMessage(CMessage* self, const Descriptor* descriptor, const FieldDescriptor* field_descriptor) { - MessageFactory* message_factory = GetDescriptorPool()->message_factory; + MessageFactory* message_factory = GetFactoryForMessage(self); Message* released_message = self->message->GetReflection()->ReleaseMessage( self->message, field_descriptor, message_factory); // ReleaseMessage will return NULL which differs from @@ -1883,8 +1921,8 @@ static PyObject* MergeFromString(CMessage* self, PyObject* arg) { AssureWritable(self); io::CodedInputStream input( reinterpret_cast(data), data_length); - input.SetExtensionRegistry(GetDescriptorPool()->pool, - GetDescriptorPool()->message_factory); + PyDescriptorPool* pool = GetDescriptorPoolForMessage(self); + input.SetExtensionRegistry(pool->pool, pool->message_factory); bool success = self->message->MergePartialFromCodedStream(&input); if (success) { return PyInt_FromLong(input.CurrentPosition()); @@ -1907,11 +1945,6 @@ static PyObject* ByteSize(CMessage* self, PyObject* args) { static PyObject* RegisterExtension(PyObject* cls, PyObject* extension_handle) { - ScopedPyObjectPtr message_descriptor(PyObject_GetAttr(cls, kDESCRIPTOR)); - if (message_descriptor == NULL) { - return NULL; - } - const FieldDescriptor* descriptor = GetExtensionDescriptor(extension_handle); if (descriptor == NULL) { @@ -1920,13 +1953,6 @@ static PyObject* RegisterExtension(PyObject* cls, const Descriptor* cmessage_descriptor = GetMessageDescriptor( reinterpret_cast(cls)); - if (cmessage_descriptor != descriptor->containing_type()) { - if (PyObject_SetAttrString(extension_handle, "containing_type", - message_descriptor) < 0) { - return NULL; - } - } - ScopedPyObjectPtr extensions_by_name( PyObject_GetAttr(cls, k_extensions_by_name)); if (extensions_by_name == NULL) { @@ -2050,7 +2076,8 @@ static PyObject* ListFields(CMessage* self) { // TODO(amauryfa): consider building the class on the fly! if (fields[i]->message_type() != NULL && cdescriptor_pool::GetMessageClass( - GetDescriptorPool(), fields[i]->message_type()) == NULL) { + GetDescriptorPoolForMessage(self), + fields[i]->message_type()) == NULL) { PyErr_Clear(); continue; } @@ -2207,7 +2234,9 @@ PyObject* InternalGetScalar(const Message* message, message->GetReflection()->GetUnknownFields(*message); for (int i = 0; i < unknown_field_set.field_count(); ++i) { if (unknown_field_set.field(i).number() == - field_descriptor->number()) { + field_descriptor->number() && + unknown_field_set.field(i).type() == + google::protobuf::UnknownField::TYPE_VARINT) { result = PyInt_FromLong(unknown_field_set.field(i).varint()); break; } @@ -2233,11 +2262,12 @@ PyObject* InternalGetScalar(const Message* message, PyObject* InternalGetSubMessage( CMessage* self, const FieldDescriptor* field_descriptor) { const Reflection* reflection = self->message->GetReflection(); + PyDescriptorPool* pool = GetDescriptorPoolForMessage(self); const Message& sub_message = reflection->GetMessage( - *self->message, field_descriptor, GetDescriptorPool()->message_factory); + *self->message, field_descriptor, pool->message_factory); PyObject *message_class = cdescriptor_pool::GetMessageClass( - GetDescriptorPool(), field_descriptor->message_type()); + pool, field_descriptor->message_type()); if (message_class == NULL) { return NULL; } @@ -2560,7 +2590,7 @@ PyObject* GetAttr(CMessage* self, PyObject* name) { const FieldDescriptor* value_type = entry_type->FindFieldByName("value"); if (value_type->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { PyObject* value_class = cdescriptor_pool::GetMessageClass( - GetDescriptorPool(), value_type->message_type()); + GetDescriptorPoolForMessage(self), value_type->message_type()); if (value_class == NULL) { return NULL; } @@ -2583,7 +2613,7 @@ PyObject* GetAttr(CMessage* self, PyObject* name) { PyObject* py_container = NULL; if (field_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { PyObject *message_class = cdescriptor_pool::GetMessageClass( - GetDescriptorPool(), field_descriptor->message_type()); + GetDescriptorPoolForMessage(self), field_descriptor->message_type()); if (message_class == NULL) { return NULL; } @@ -2908,9 +2938,10 @@ bool InitProto2MessageModule(PyObject *m) { // Expose the DescriptorPool used to hold all descriptors added from generated // pb2.py files. - Py_INCREF(GetDescriptorPool()); // PyModule_AddObject steals a reference. - PyModule_AddObject( - m, "default_pool", reinterpret_cast(GetDescriptorPool())); + // PyModule_AddObject steals a reference. + Py_INCREF(GetDefaultDescriptorPool()); + PyModule_AddObject(m, "default_pool", + reinterpret_cast(GetDefaultDescriptorPool())); // This implementation provides full Descriptor types, we advertise it so that // descriptor.py can use them in replacement of the Python classes. diff --git a/python/google/protobuf/pyext/message.h b/python/google/protobuf/pyext/message.h index f147d433ef..1ff82e2f64 100644 --- a/python/google/protobuf/pyext/message.h +++ b/python/google/protobuf/pyext/message.h @@ -49,12 +49,15 @@ class Message; class Reflection; class FieldDescriptor; class Descriptor; +class DescriptorPool; +class MessageFactory; using internal::shared_ptr; namespace python { struct ExtensionDict; +struct PyDescriptorPool; typedef struct CMessage { PyObject_HEAD; @@ -220,6 +223,16 @@ PyObject* FindInitializationErrors(CMessage* self); int SetOwner(CMessage* self, const shared_ptr& new_owner); int AssureWritable(CMessage* self); + +// Returns the "best" DescriptorPool for the given message. +// This is often equivalent to message.DESCRIPTOR.pool, but not always, when +// the message class was created from a MessageFactory using a custom pool which +// uses the generated pool as an underlay. +// +// The returned pool is suitable for finding fields and building submessages, +// even in the case of extensions. +PyDescriptorPool* GetDescriptorPoolForMessage(CMessage* message); + } // namespace cmessage diff --git a/python/google/protobuf/symbol_database.py b/python/google/protobuf/symbol_database.py index 4c70b3938d..b81ef4d78c 100644 --- a/python/google/protobuf/symbol_database.py +++ b/python/google/protobuf/symbol_database.py @@ -60,6 +60,7 @@ Example usage: """ +from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool @@ -72,6 +73,31 @@ class SymbolDatabase(object): buffer types used within a program. """ + # pylint: disable=protected-access + if _descriptor._USE_C_DESCRIPTORS: + + def __new__(cls): + raise TypeError("Instances of SymbolDatabase cannot be created") + + @classmethod + def _CreateDefaultDatabase(cls): + self = object.__new__(cls) # Bypass the __new__ above. + # Don't call __init__() and initialize here. + self._symbols = {} + self._symbols_by_file = {} + # As of today all descriptors are registered and retrieved from + # _message.default_pool (see FileDescriptor.__new__), so it's not + # necessary to use another pool. + self.pool = _descriptor._message.default_pool + return self + # pylint: enable=protected-access + + else: + + @classmethod + def _CreateDefaultDatabase(cls): + return cls() + def __init__(self): """Constructor.""" @@ -177,7 +203,7 @@ class SymbolDatabase(object): result.update(self._symbols_by_file[f]) return result -_DEFAULT = SymbolDatabase() +_DEFAULT = SymbolDatabase._CreateDefaultDatabase() def Default(): diff --git a/python/google/protobuf/text_encoding.py b/python/google/protobuf/text_encoding.py index a0728e3ca8..9899563825 100644 --- a/python/google/protobuf/text_encoding.py +++ b/python/google/protobuf/text_encoding.py @@ -27,6 +27,7 @@ # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + """Encoding related utilities.""" import re diff --git a/python/google/protobuf/text_format.py b/python/google/protobuf/text_format.py index 1399223fbf..e4fadf09e8 100755 --- a/python/google/protobuf/text_format.py +++ b/python/google/protobuf/text_format.py @@ -28,9 +28,17 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Copyright 2007 Google Inc. All Rights Reserved. +"""Contains routines for printing protocol messages in text format. -"""Contains routines for printing protocol messages in text format.""" +Simple usage example: + + # Create a proto object and serialize it to a text proto string. + message = my_proto_pb2.MyMessage(foo='bar') + text_proto = text_format.MessageToString(message) + + # Parse a text proto string. + message = text_format.Parse(text_proto, my_proto_pb2.MyMessage()) +""" __author__ = 'kenton@google.com (Kenton Varda)' diff --git a/python/google/protobuf/util/__init__.py b/python/google/protobuf/util/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python/setup.py b/python/setup.py index 8269c4a5f4..a7bc6cd6b2 100755 --- a/python/setup.py +++ b/python/setup.py @@ -88,6 +88,15 @@ def GenerateUnittestProtos(): generate_proto("../src/google/protobuf/unittest_mset_wire_format.proto", False) generate_proto("../src/google/protobuf/unittest_no_generic_services.proto", False) generate_proto("../src/google/protobuf/unittest_proto3_arena.proto", False) + # Move the well-known-types proto to required when they are no longer only + # required by json format tests. + generate_proto("../src/google/protobuf/timestamp.proto", False) + generate_proto("../src/google/protobuf/duration.proto", False) + generate_proto("../src/google/protobuf/wrappers.proto", False) + generate_proto("../src/google/protobuf/struct.proto", False) + generate_proto("../src/google/protobuf/any.proto", False) + generate_proto("../src/google/protobuf/field_mask.proto", False) + generate_proto("../src/google/protobuf/util/json_format_proto3.proto", False) generate_proto("google/protobuf/internal/descriptor_pool_test1.proto", False) generate_proto("google/protobuf/internal/descriptor_pool_test2.proto", False) generate_proto("google/protobuf/internal/factory_test1.proto", False) diff --git a/src/google/protobuf/any.cc b/src/google/protobuf/any.cc index c6ed37ae38..7351d3773b 100644 --- a/src/google/protobuf/any.cc +++ b/src/google/protobuf/any.cc @@ -43,6 +43,7 @@ string GetTypeUrl(const Descriptor* message) { const char kAnyFullTypeName[] = "google.protobuf.Any"; const char kTypeGoogleApisComPrefix[] = "type.googleapis.com/"; +const char kTypeGoogleProdComPrefix[] = "type.googleprod.com/"; AnyMetadata::AnyMetadata(UrlType* type_url, ValueType* value) : type_url_(type_url), value_(value) { @@ -70,11 +71,17 @@ bool AnyMetadata::InternalIs(const Descriptor* descriptor) const { } bool ParseAnyTypeUrl(const string& type_url, string* full_type_name) { - const int prefix_len = strlen(kTypeGoogleApisComPrefix); - if (strncmp(type_url.c_str(), kTypeGoogleApisComPrefix, prefix_len) == 0) { - full_type_name->assign(type_url.data() + prefix_len, - type_url.size() - prefix_len); - return true; + static const char* prefix[] = { + kTypeGoogleApisComPrefix, + kTypeGoogleProdComPrefix + }; + for (int i = 0; i < 2; i++) { + const int prefix_len = strlen(prefix[i]); + if (strncmp(type_url.c_str(), prefix[i], prefix_len) == 0) { + full_type_name->assign(type_url.data() + prefix_len, + type_url.size() - prefix_len); + return true; + } } return false; } diff --git a/src/google/protobuf/any.h b/src/google/protobuf/any.h index 7eeb6b707a..f760ad5d24 100644 --- a/src/google/protobuf/any.h +++ b/src/google/protobuf/any.h @@ -70,6 +70,7 @@ class LIBPROTOBUF_EXPORT AnyMetadata { extern const char kAnyFullTypeName[]; // "google.protobuf.Any". extern const char kTypeGoogleApisComPrefix[]; // "type.googleapis.com/". +extern const char kTypeGoogleProdComPrefix[]; // "type.googleprod.com/". // Get the proto type name from Any::type_url value. For example, passing // "type.googleapis.com/rpc.QueryOrigin" will return "rpc.QueryOrigin" in diff --git a/src/google/protobuf/arena.cc b/src/google/protobuf/arena.cc index 907a6a20c5..74b0965520 100755 --- a/src/google/protobuf/arena.cc +++ b/src/google/protobuf/arena.cc @@ -38,17 +38,17 @@ namespace google { namespace protobuf { google::protobuf::internal::SequenceNumber Arena::lifecycle_id_generator_; -#ifdef PROTOBUF_USE_DLLS -Arena::ThreadCache& Arena::thread_cache() { - static GOOGLE_THREAD_LOCAL ThreadCache thread_cache_ = { -1, NULL }; - return thread_cache_; -} -#elif defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) +#if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) Arena::ThreadCache& Arena::thread_cache() { static internal::ThreadLocalStorage* thread_cache_ = new internal::ThreadLocalStorage(); return *thread_cache_->Get(); } +#elif defined(PROTOBUF_USE_DLLS) +Arena::ThreadCache& Arena::thread_cache() { + static GOOGLE_THREAD_LOCAL ThreadCache thread_cache_ = { -1, NULL }; + return thread_cache_; +} #else GOOGLE_THREAD_LOCAL Arena::ThreadCache Arena::thread_cache_ = { -1, NULL }; #endif diff --git a/src/google/protobuf/arena.h b/src/google/protobuf/arena.h index 074a9e5411..16e0d50edb 100644 --- a/src/google/protobuf/arena.h +++ b/src/google/protobuf/arena.h @@ -38,7 +38,16 @@ #if __cplusplus >= 201103L #include #endif +#if defined(_MSC_VER) && !_HAS_EXCEPTIONS +// Work around bugs in MSVC header when _HAS_EXCEPTIONS=0. +#include #include +namespace std { +using type_info = ::type_info; +} +#else +#include +#endif #include #include @@ -533,15 +542,15 @@ class LIBPROTOBUF_EXPORT Arena { static const size_t kHeaderSize = sizeof(Block); static google::protobuf::internal::SequenceNumber lifecycle_id_generator_; -#ifdef PROTOBUF_USE_DLLS - // Thread local variables cannot be exposed through DLL interface but we can - // wrap them in static functions. - static ThreadCache& thread_cache(); -#elif defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) +#if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) // Android ndk does not support GOOGLE_THREAD_LOCAL keyword so we use a custom thread // local storage class we implemented. // iOS also does not support the GOOGLE_THREAD_LOCAL keyword. static ThreadCache& thread_cache(); +#elif defined(PROTOBUF_USE_DLLS) + // Thread local variables cannot be exposed through DLL interface but we can + // wrap them in static functions. + static ThreadCache& thread_cache(); #else static GOOGLE_THREAD_LOCAL ThreadCache thread_cache_; static ThreadCache& thread_cache() { return thread_cache_; } @@ -581,11 +590,13 @@ class LIBPROTOBUF_EXPORT Arena { template static double DestructorSkippable(...); + // The raw_skippable_value const bool variable is separated from the typedef + // line below as a work-around of an NVCC 7.0 (and earlier) compiler bug. + static const bool raw_skippable_value = + sizeof(DestructorSkippable(static_cast(0))) == + sizeof(char) || google::protobuf::internal::has_trivial_destructor::value == true; // This will resolve to either google::protobuf::internal::true_type or google::protobuf::internal::false_type. - typedef google::protobuf::internal::integral_constant(static_cast(0))) == - sizeof(char) || google::protobuf::internal::has_trivial_destructor::value == true> - type; + typedef google::protobuf::internal::integral_constant type; static const type value; }; diff --git a/src/google/protobuf/arena_unittest.cc b/src/google/protobuf/arena_unittest.cc index c9ca1fd18d..6b67f446fd 100644 --- a/src/google/protobuf/arena_unittest.cc +++ b/src/google/protobuf/arena_unittest.cc @@ -362,7 +362,7 @@ TEST(ArenaTest, ReleaseMessage) { Arena arena; TestAllTypes* arena_message = Arena::CreateMessage(&arena); arena_message->mutable_optional_nested_message()->set_bb(118); - scoped_ptr nested( + google::protobuf::scoped_ptr nested( arena_message->release_optional_nested_message()); EXPECT_EQ(118, nested->bb()); @@ -383,7 +383,7 @@ TEST(ArenaTest, ReleaseString) { Arena arena; TestAllTypes* arena_message = Arena::CreateMessage(&arena); arena_message->set_optional_string("hello"); - scoped_ptr released_str( + google::protobuf::scoped_ptr released_str( arena_message->release_optional_string()); EXPECT_EQ("hello", *released_str); diff --git a/src/google/protobuf/compiler/command_line_interface_unittest.cc b/src/google/protobuf/compiler/command_line_interface_unittest.cc index 9560d0e0b4..46ea5c4e42 100644 --- a/src/google/protobuf/compiler/command_line_interface_unittest.cc +++ b/src/google/protobuf/compiler/command_line_interface_unittest.cc @@ -63,13 +63,13 @@ #include #include -namespace google { -namespace protobuf { -namespace compiler { // Disable the whole test when we use tcmalloc for "draconian" heap checks, in // which case tcmalloc will print warnings that fail the plugin tests. #if !GOOGLE_PROTOBUF_HEAP_CHECK_DRACONIAN +namespace google { +namespace protobuf { +namespace compiler { #if defined(_WIN32) #ifndef STDIN_FILENO @@ -1800,8 +1800,8 @@ TEST_F(EncodeDecodeTest, ProtoParseError) { } // anonymous namespace -#endif // !GOOGLE_PROTOBUF_HEAP_CHECK_DRACONIAN - } // namespace compiler } // namespace protobuf + +#endif // !GOOGLE_PROTOBUF_HEAP_CHECK_DRACONIAN } // namespace google diff --git a/src/google/protobuf/compiler/cpp/cpp_map_field.cc b/src/google/protobuf/compiler/cpp/cpp_map_field.cc index 25acc61bed..e5e2f07dd4 100644 --- a/src/google/protobuf/compiler/cpp/cpp_map_field.cc +++ b/src/google/protobuf/compiler/cpp/cpp_map_field.cc @@ -66,7 +66,7 @@ void SetMessageVariables(const FieldDescriptor* descriptor, (*variables)["wrapper"] = "EntryWrapper"; break; case FieldDescriptor::CPPTYPE_ENUM: - (*variables)["val_cpp"] = ClassName(val->enum_type(), false); + (*variables)["val_cpp"] = ClassName(val->enum_type(), true); (*variables)["wrapper"] = "EnumEntryWrapper"; break; default: @@ -200,7 +200,7 @@ GenerateMergeFromCodedStream(io::Printer* printer) const { case FieldDescriptor::CPPTYPE_ENUM: printer->Print(variables_, "(*mutable_$name$())[entry->key()] =\n" - " static_cast<$val_cpp$>(*entry->mutable_value());\n"); + " static_cast< $val_cpp$ >(*entry->mutable_value());\n"); break; default: printer->Print(variables_, @@ -215,7 +215,7 @@ GenerateMergeFromCodedStream(io::Printer* printer) const { " DO_(entry->ParseFromString(data));\n" " if ($val_cpp$_IsValid(*entry->mutable_value())) {\n" " (*mutable_$name$())[entry->key()] =\n" - " static_cast<$val_cpp$>(*entry->mutable_value());\n" + " static_cast< $val_cpp$ >(*entry->mutable_value());\n" " } else {\n"); if (HasDescriptorMethods(descriptor_->file())) { printer->Print(variables_, diff --git a/src/google/protobuf/compiler/cpp/cpp_message.cc b/src/google/protobuf/compiler/cpp/cpp_message.cc index fc1ce962ab..8cc8c7bae2 100644 --- a/src/google/protobuf/compiler/cpp/cpp_message.cc +++ b/src/google/protobuf/compiler/cpp/cpp_message.cc @@ -350,7 +350,7 @@ void CollectMapInfo(const Descriptor* descriptor, (*variables)["val"] = FieldMessageTypeName(val); break; case FieldDescriptor::CPPTYPE_ENUM: - (*variables)["val"] = ClassName(val->enum_type(), false); + (*variables)["val"] = ClassName(val->enum_type(), true); break; default: (*variables)["val"] = PrimitiveTypeName(val->cpp_type()); diff --git a/src/google/protobuf/compiler/java/java_enum_field_lite.cc b/src/google/protobuf/compiler/java/java_enum_field_lite.cc index 2c3608c290..e8bf15d0d5 100644 --- a/src/google/protobuf/compiler/java/java_enum_field_lite.cc +++ b/src/google/protobuf/compiler/java/java_enum_field_lite.cc @@ -299,7 +299,7 @@ GenerateParsingCode(io::Printer* printer) const { "if (value == null) {\n"); if (PreserveUnknownFields(descriptor_->containing_type())) { printer->Print(variables_, - " unknownFields.mergeVarintField($number$, rawValue);\n"); + " super.mergeVarintField($number$, rawValue);\n"); } printer->Print(variables_, "} else {\n" @@ -492,7 +492,7 @@ GenerateParsingCode(io::Printer* printer) const { "if (value == null) {\n"); if (PreserveUnknownFields(descriptor_->containing_type())) { printer->Print(variables_, - " unknownFields.mergeVarintField($number$, rawValue);\n"); + " super.mergeVarintField($number$, rawValue);\n"); } printer->Print(variables_, "} else {\n" @@ -850,7 +850,7 @@ GenerateParsingCode(io::Printer* printer) const { "if (value == null) {\n"); if (PreserveUnknownFields(descriptor_->containing_type())) { printer->Print(variables_, - " unknownFields.mergeVarintField($number$, rawValue);\n"); + " super.mergeVarintField($number$, rawValue);\n"); } printer->Print(variables_, "} else {\n" diff --git a/src/google/protobuf/compiler/java/java_map_field_lite.cc b/src/google/protobuf/compiler/java/java_map_field_lite.cc index 4fe656d3bb..d2039403e9 100644 --- a/src/google/protobuf/compiler/java/java_map_field_lite.cc +++ b/src/google/protobuf/compiler/java/java_map_field_lite.cc @@ -405,7 +405,7 @@ GenerateParsingCode(io::Printer* printer) const { printer->Print( variables_, "if ($value_enum_type$.valueOf($name$.getValue()) == null) {\n" - " unknownFields.mergeLengthDelimitedField($number$, bytes);\n" + " super.mergeLengthDelimitedField($number$, bytes);\n" "} else {\n" " $name$_.getMutableMap().put($name$.getKey(), $name$.getValue());\n" "}\n"); diff --git a/src/google/protobuf/compiler/java/java_message_lite.cc b/src/google/protobuf/compiler/java/java_message_lite.cc index 8b6c75b8ba..94ed2c396a 100644 --- a/src/google/protobuf/compiler/java/java_message_lite.cc +++ b/src/google/protobuf/compiler/java/java_message_lite.cc @@ -1029,12 +1029,6 @@ GenerateParsingConstructor(io::Printer* printer) { "bit_field_name", GetBitFieldName(i)); } - if (PreserveUnknownFields(descriptor_)) { - printer->Print( - "com.google.protobuf.UnknownFieldSetLite.Builder unknownFields =\n" - " com.google.protobuf.UnknownFieldSetLite.newBuilder();\n"); - } - printer->Print( "try {\n"); printer->Indent(); @@ -1056,13 +1050,10 @@ GenerateParsingConstructor(io::Printer* printer) { if (PreserveUnknownFields(descriptor_)) { if (descriptor_->extension_range_count() > 0) { - // Lite runtime directly invokes parseUnknownField to reduce method - // counts. printer->Print( "default: {\n" - " if (!parseUnknownField(extensions, getDefaultInstanceForType(),\n" - " input, unknownFields,\n" - " extensionRegistry, tag)) {\n" + " if (!parseUnknownField(getDefaultInstanceForType(),\n" + " input, extensionRegistry, tag)) {\n" " done = true;\n" // it's an endgroup tag " }\n" " break;\n" @@ -1070,8 +1061,7 @@ GenerateParsingConstructor(io::Printer* printer) { } else { printer->Print( "default: {\n" - " if (!parseUnknownField(input, unknownFields,\n" - " extensionRegistry, tag)) {\n" + " if (!parseUnknownField(tag, input)) {\n" " done = true;\n" // it's an endgroup tag " }\n" " break;\n" @@ -1146,16 +1136,8 @@ GenerateParsingConstructor(io::Printer* printer) { field_generators_.get(field).GenerateParsingDoneCode(printer); } - if (PreserveUnknownFields(descriptor_)) { - // Make unknown fields immutable. - printer->Print("this.unknownFields = unknownFields.build();\n"); - } - - if (descriptor_->extension_range_count() > 0) { - // Make extensions immutable. - printer->Print( - "makeExtensionsImmutable(extensions);\n"); - } + printer->Print( + "doneParsing();\n"); printer->Outdent(); printer->Outdent(); diff --git a/src/google/protobuf/compiler/parser.cc b/src/google/protobuf/compiler/parser.cc index 4d0184252b..a389a4fcee 100644 --- a/src/google/protobuf/compiler/parser.cc +++ b/src/google/protobuf/compiler/parser.cc @@ -993,6 +993,9 @@ bool Parser::ParseFieldOptions(FieldDescriptorProto* field, // We intentionally pass field_location rather than location here, since // the default value is not actually an option. DO(ParseDefaultAssignment(field, field_location, containing_file)); + } else if (LookingAt("json_name")) { + // Like default value, this "json_name" is not an actual option. + DO(ParseJsonName(field, field_location, containing_file)); } else { DO(ParseOption(field->mutable_options(), location, containing_file, OPTION_ASSIGNMENT)); @@ -1140,6 +1143,28 @@ bool Parser::ParseDefaultAssignment( return true; } +bool Parser::ParseJsonName( + FieldDescriptorProto* field, + const LocationRecorder& field_location, + const FileDescriptorProto* containing_file) { + if (field->has_json_name()) { + AddError("Already set option \"json_name\"."); + field->clear_json_name(); + } + + DO(Consume("json_name")); + DO(Consume("=")); + + LocationRecorder location(field_location, + FieldDescriptorProto::kJsonNameFieldNumber); + location.RecordLegacyLocation( + field, DescriptorPool::ErrorCollector::OPTION_VALUE); + DO(ConsumeString(field->mutable_json_name(), + "Expected string for JSON name.")); + return true; +} + + bool Parser::ParseOptionNamePart(UninterpretedOption* uninterpreted_option, const LocationRecorder& part_location, const FileDescriptorProto* containing_file) { diff --git a/src/google/protobuf/compiler/parser.h b/src/google/protobuf/compiler/parser.h index 007b001c35..3ba1e17077 100644 --- a/src/google/protobuf/compiler/parser.h +++ b/src/google/protobuf/compiler/parser.h @@ -439,6 +439,10 @@ class LIBPROTOBUF_EXPORT Parser { const LocationRecorder& field_location, const FileDescriptorProto* containing_file); + bool ParseJsonName(FieldDescriptorProto* field, + const LocationRecorder& field_location, + const FileDescriptorProto* containing_file); + enum OptionStyle { OPTION_ASSIGNMENT, // just "name = value" OPTION_STATEMENT // "option name = value;" diff --git a/src/google/protobuf/compiler/parser_unittest.cc b/src/google/protobuf/compiler/parser_unittest.cc index cc6f1efb21..0d729e0b99 100644 --- a/src/google/protobuf/compiler/parser_unittest.cc +++ b/src/google/protobuf/compiler/parser_unittest.cc @@ -452,6 +452,20 @@ TEST_F(ParseMessageTest, FieldDefaults) { #undef ETC } +TEST_F(ParseMessageTest, FieldJsonName) { + ExpectParsesTo( + "message TestMessage {\n" + " optional string foo = 1 [json_name = \"@type\"];\n" + "}\n", + "message_type {" + " name: \"TestMessage\"" + " field {\n" + " name: \"foo\" label: LABEL_OPTIONAL type: TYPE_STRING number: 1" + " json_name: \"@type\"\n" + " }\n" + "}\n"); +} + TEST_F(ParseMessageTest, FieldOptions) { ExpectParsesTo( "message TestMessage {\n" @@ -1126,6 +1140,22 @@ TEST_F(ParseErrorTest, DefaultValueTooLarge) { "6:36: Integer out of range.\n"); } +TEST_F(ParseErrorTest, JsonNameNotString) { + ExpectHasErrors( + "message TestMessage {\n" + " optional string foo = 1 [json_name=1];\n" + "}\n", + "1:37: Expected string for JSON name.\n"); +} + +TEST_F(ParseErrorTest, DuplicateJsonName) { + ExpectHasErrors( + "message TestMessage {\n" + " optional uint32 foo = 1 [json_name=\"a\",json_name=\"b\"];\n" + "}\n", + "1:41: Already set option \"json_name\".\n"); +} + TEST_F(ParseErrorTest, EnumValueOutOfRange) { ExpectHasErrors( "enum TestEnum {\n" diff --git a/src/google/protobuf/compiler/python/python_generator.cc b/src/google/protobuf/compiler/python/python_generator.cc index e81af7009d..4d500f90e9 100644 --- a/src/google/protobuf/compiler/python/python_generator.cc +++ b/src/google/protobuf/compiler/python/python_generator.cc @@ -28,6 +28,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +//#PY25 compatible generated code for GAE. // Copyright 2007 Google Inc. All Rights Reserved. // Author: robinson@google.com (Will Robinson) // @@ -166,6 +167,7 @@ void PrintTopBoilerplate( printer->Print( "# Generated by the protocol buffer compiler. DO NOT EDIT!\n" "# source: $filename$\n" + "\nimport sys\n_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))" //##PY25 "\n", "filename", file->name()); if (HasTopLevelEnums(file)) { @@ -257,9 +259,12 @@ string StringifyDefaultValue(const FieldDescriptor& field) { case FieldDescriptor::CPPTYPE_ENUM: return SimpleItoa(field.default_value_enum()->number()); case FieldDescriptor::CPPTYPE_STRING: - return "b\"" + CEscape(field.default_value_string()) + - (field.type() != FieldDescriptor::TYPE_STRING ? "\"" : - "\".decode('utf-8')"); +//##!PY25 return "b\"" + CEscape(field.default_value_string()) + +//##!PY25 (field.type() != FieldDescriptor::TYPE_STRING ? "\"" : +//##!PY25 "\".decode('utf-8')"); + return "_b(\"" + CEscape(field.default_value_string()) + //##PY25 + (field.type() != FieldDescriptor::TYPE_STRING ? "\")" : //##PY25 + "\").decode('utf-8')"); //##PY25 case FieldDescriptor::CPPTYPE_MESSAGE: return "None"; } @@ -385,7 +390,8 @@ void Generator::PrintFileDescriptor() const { printer_->Print(m, file_descriptor_template); printer_->Indent(); printer_->Print( - "serialized_pb=b'$value$'\n", +//##!PY25 "serialized_pb=b'$value$'\n", + "serialized_pb=_b('$value$')\n", //##PY25 "value", strings::CHexEscape(file_descriptor_serialized_)); if (file_->dependency_count() != 0) { printer_->Print(",\ndependencies=["); @@ -1029,8 +1035,10 @@ string Generator::OptionsValue( return "None"; } else { string full_class_name = "descriptor_pb2." + class_name; - return "_descriptor._ParseOptions(" + full_class_name + "(), b'" - + CEscape(serialized_options)+ "')"; +//##!PY25 return "_descriptor._ParseOptions(" + full_class_name + "(), b'" +//##!PY25 + CEscape(serialized_options)+ "')"; + return "_descriptor._ParseOptions(" + full_class_name + "(), _b('" //##PY25 + + CEscape(serialized_options)+ "'))"; //##PY25 } } diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 5256b83c44..b40fe95a91 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -34,6 +34,10 @@ #include #include +#include +#ifndef _SHARED_PTR_H +#include +#endif #include #include #include @@ -1726,6 +1730,20 @@ void FileDescriptor::CopyTo(FileDescriptorProto* proto) const { } } +void FileDescriptor::CopyJsonNameTo(FileDescriptorProto* proto) const { + if (message_type_count() != proto->message_type_size() || + extension_count() != proto->extension_size()) { + GOOGLE_LOG(ERROR) << "Cannot copy json_name to a proto of a different size."; + return; + } + for (int i = 0; i < message_type_count(); i++) { + message_type(i)->CopyJsonNameTo(proto->mutable_message_type(i)); + } + for (int i = 0; i < extension_count(); i++) { + extension(i)->CopyJsonNameTo(proto->mutable_extension(i)); + } +} + void FileDescriptor::CopySourceCodeInfoTo(FileDescriptorProto* proto) const { if (source_code_info_ && source_code_info_ != &SourceCodeInfo::default_instance()) { @@ -1770,9 +1788,30 @@ void Descriptor::CopyTo(DescriptorProto* proto) const { } } +void Descriptor::CopyJsonNameTo(DescriptorProto* proto) const { + if (field_count() != proto->field_size() || + nested_type_count() != proto->nested_type_size() || + extension_count() != proto->extension_size()) { + GOOGLE_LOG(ERROR) << "Cannot copy json_name to a proto of a different size."; + return; + } + for (int i = 0; i < field_count(); i++) { + field(i)->CopyJsonNameTo(proto->mutable_field(i)); + } + for (int i = 0; i < nested_type_count(); i++) { + nested_type(i)->CopyJsonNameTo(proto->mutable_nested_type(i)); + } + for (int i = 0; i < extension_count(); i++) { + extension(i)->CopyJsonNameTo(proto->mutable_extension(i)); + } +} + void FieldDescriptor::CopyTo(FieldDescriptorProto* proto) const { proto->set_name(name()); proto->set_number(number()); + if (has_json_name_) { + proto->set_json_name(json_name()); + } // Some compilers do not allow static_cast directly between two enum types, // so we must cast to int first. @@ -1819,6 +1858,10 @@ void FieldDescriptor::CopyTo(FieldDescriptorProto* proto) const { } } +void FieldDescriptor::CopyJsonNameTo(FieldDescriptorProto* proto) const { + proto->set_json_name(json_name()); +} + void OneofDescriptor::CopyTo(OneofDescriptorProto* proto) const { proto->set_name(name()); } @@ -4136,6 +4179,14 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, tables_->AllocateString(ToCamelCase(proto.name(), /* lower_first = */ true)); + if (proto.has_json_name()) { + result->has_json_name_ = true; + result->json_name_ = tables_->AllocateString(proto.json_name()); + } else { + result->has_json_name_ = false; + result->json_name_ = result->camelcase_name_; + } + // Some compilers do not allow static_cast directly between two enum types, // so we must cast to int first. result->type_ = static_cast( @@ -5040,6 +5091,20 @@ void DescriptorBuilder::ValidateProto3( } } +static string ToLowercaseWithoutUnderscores(const string& name) { + string result; + for (int i = 0; i < name.size(); ++i) { + if (name[i] != '_') { + if (name[i] >= 'A' && name[i] <= 'Z') { + result.push_back(name[i] - 'A' + 'a'); + } else { + result.push_back(name[i]); + } + } + } + return result; +} + void DescriptorBuilder::ValidateProto3Message( Descriptor* message, const DescriptorProto& proto) { for (int i = 0; i < message->nested_type_count(); ++i) { @@ -5067,6 +5132,25 @@ void DescriptorBuilder::ValidateProto3Message( DescriptorPool::ErrorCollector::OTHER, "MessageSet is not supported in proto3."); } + + // In proto3, we reject field names if they conflict in camelCase. + // Note that we currently enforce a stricter rule: Field names must be + // unique after being converted to lowercase with underscores removed. + map name_to_field; + for (int i = 0; i < message->field_count(); ++i) { + string lowercase_name = ToLowercaseWithoutUnderscores( + message->field(i)->name()); + if (name_to_field.find(lowercase_name) != name_to_field.end()) { + AddError(message->full_name(), proto, + DescriptorPool::ErrorCollector::OTHER, + "The JSON camcel-case name of field \"" + + message->field(i)->name() + "\" conflicts with field \"" + + name_to_field[lowercase_name]->name() + "\". This is not " + + "allowed in proto3."); + } else { + name_to_field[lowercase_name] = message->field(i); + } + } } void DescriptorBuilder::ValidateProto3Field( @@ -5602,7 +5686,7 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( // First set the value on the UnknownFieldSet corresponding to the // innermost message. - scoped_ptr unknown_fields(new UnknownFieldSet()); + google::protobuf::scoped_ptr unknown_fields(new UnknownFieldSet()); if (!SetOptionValue(field, unknown_fields.get())) { return false; // SetOptionValue() already added the error. } @@ -5612,7 +5696,8 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( for (vector::reverse_iterator iter = intermediate_fields.rbegin(); iter != intermediate_fields.rend(); ++iter) { - scoped_ptr parent_unknown_fields(new UnknownFieldSet()); + google::protobuf::scoped_ptr parent_unknown_fields( + new UnknownFieldSet()); switch ((*iter)->type()) { case FieldDescriptor::TYPE_MESSAGE: { io::StringOutputStream outstr( @@ -5998,7 +6083,7 @@ bool DescriptorBuilder::OptionInterpreter::SetAggregateOption( } const Descriptor* type = option_field->message_type(); - scoped_ptr dynamic(dynamic_factory_.GetPrototype(type)->New()); + google::protobuf::scoped_ptr dynamic(dynamic_factory_.GetPrototype(type)->New()); GOOGLE_CHECK(dynamic.get() != NULL) << "Could not create an instance of " << option_field->DebugString(); diff --git a/src/google/protobuf/descriptor.h b/src/google/protobuf/descriptor.h index 2ab316a521..e7e8c6af42 100644 --- a/src/google/protobuf/descriptor.h +++ b/src/google/protobuf/descriptor.h @@ -54,6 +54,10 @@ #ifndef GOOGLE_PROTOBUF_DESCRIPTOR_H__ #define GOOGLE_PROTOBUF_DESCRIPTOR_H__ +#include +#ifndef _SHARED_PTR_H +#include +#endif #include #include #include @@ -111,8 +115,17 @@ class UnknownField; // Defined in generated_message_reflection.h. namespace internal { - class GeneratedMessageReflection; -} +class GeneratedMessageReflection; +} // namespace internal + +// Defined in command_line_interface.cc +namespace compiler { +class CommandLineInterface; +} // namespace compiler + +namespace descriptor_unittest { +class DescriptorTest; +} // namespace descriptor_unittest // NB, all indices are zero-based. struct SourceLocation { @@ -343,6 +356,12 @@ class LIBPROTOBUF_EXPORT Descriptor { private: typedef MessageOptions OptionsType; + // Allows tests to test CopyTo(proto, true). + friend class ::google::protobuf::descriptor_unittest::DescriptorTest; + + // Fill the json_name field of FieldDescriptorProto. + void CopyJsonNameTo(DescriptorProto* proto) const; + // Internal version of DebugString; controls the level of indenting for // correct depth. Takes |options| to control debug-string options, and // |include_opening_clause| to indicate whether the "message ... " part of the @@ -484,6 +503,7 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { const string& name() const; // Name of this field within the message. const string& full_name() const; // Fully-qualified name of the field. + const string& json_name() const; // JSON name of this field. const FileDescriptor* file() const;// File in which this field was defined. bool is_extension() const; // Is this an extension field? int number() const; // Declared tag number. @@ -624,6 +644,9 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { private: typedef FieldOptions OptionsType; + // Fill the json_name field of FieldDescriptorProto. + void CopyJsonNameTo(FieldDescriptorProto* proto) const; + // See Descriptor::DebugString(). enum PrintLabelFlag { PRINT_LABEL, OMIT_LABEL }; void DebugString(int depth, PrintLabelFlag print_label_flag, @@ -645,6 +668,12 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { const string* full_name_; const string* lowercase_name_; const string* camelcase_name_; + // Whether the user has specified the json_name field option in the .proto + // file. + bool has_json_name_; + // If has_json_name_ is true, it's the value specified by the user. + // Otherwise, it has the same value as lowercase_name_. + const string* json_name_; const FileDescriptor* file_; int number_; Type type_; @@ -1202,6 +1231,9 @@ class LIBPROTOBUF_EXPORT FileDescriptor { // Write the source code information of this FileDescriptor into the given // FileDescriptorProto. See CopyTo() above. void CopySourceCodeInfoTo(FileDescriptorProto* proto) const; + // Fill the json_name field of FieldDescriptorProto for all fields. Can only + // be called after CopyTo(). + void CopyJsonNameTo(FileDescriptorProto* proto) const; // See Descriptor::DebugString(). string DebugString() const; @@ -1559,7 +1591,7 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // This class contains a lot of hash maps with complicated types that // we'd like to keep out of the header. class Tables; - scoped_ptr tables_; + google::protobuf::scoped_ptr tables_; bool enforce_dependencies_; bool allow_unknown_; @@ -1618,6 +1650,7 @@ PROTOBUF_DEFINE_ACCESSOR(Descriptor, is_placeholder, bool) PROTOBUF_DEFINE_STRING_ACCESSOR(FieldDescriptor, name) PROTOBUF_DEFINE_STRING_ACCESSOR(FieldDescriptor, full_name) +PROTOBUF_DEFINE_STRING_ACCESSOR(FieldDescriptor, json_name) PROTOBUF_DEFINE_STRING_ACCESSOR(FieldDescriptor, lowercase_name) PROTOBUF_DEFINE_STRING_ACCESSOR(FieldDescriptor, camelcase_name) PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, file, const FileDescriptor*) diff --git a/src/google/protobuf/descriptor.pb.cc b/src/google/protobuf/descriptor.pb.cc index fe23c0abf6..921d2cd061 100644 --- a/src/google/protobuf/descriptor.pb.cc +++ b/src/google/protobuf/descriptor.pb.cc @@ -200,7 +200,7 @@ void protobuf_AssignDesc_google_2fprotobuf_2fdescriptor_2eproto() { GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(DescriptorProto_ReservedRange, _internal_metadata_), -1); FieldDescriptorProto_descriptor_ = file->message_type(3); - static const int FieldDescriptorProto_offsets_[9] = { + static const int FieldDescriptorProto_offsets_[10] = { GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FieldDescriptorProto, name_), GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FieldDescriptorProto, number_), GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FieldDescriptorProto, label_), @@ -209,6 +209,7 @@ void protobuf_AssignDesc_google_2fprotobuf_2fdescriptor_2eproto() { GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FieldDescriptorProto, extendee_), GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FieldDescriptorProto, default_value_), GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FieldDescriptorProto, oneof_index_), + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FieldDescriptorProto, json_name_), GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(FieldDescriptorProto, options_), }; FieldDescriptorProto_reflection_ = @@ -663,101 +664,101 @@ void protobuf_AddDesc_google_2fprotobuf_2fdescriptor_2eproto() { "tobuf.DescriptorProto.ReservedRange\022\025\n\rr" "eserved_name\030\n \003(\t\032,\n\016ExtensionRange\022\r\n\005" "start\030\001 \001(\005\022\013\n\003end\030\002 \001(\005\032+\n\rReservedRang" - "e\022\r\n\005start\030\001 \001(\005\022\013\n\003end\030\002 \001(\005\"\251\005\n\024FieldD" + "e\022\r\n\005start\030\001 \001(\005\022\013\n\003end\030\002 \001(\005\"\274\005\n\024FieldD" "escriptorProto\022\014\n\004name\030\001 \001(\t\022\016\n\006number\030\003" " \001(\005\022:\n\005label\030\004 \001(\0162+.google.protobuf.Fi" "eldDescriptorProto.Label\0228\n\004type\030\005 \001(\0162*" ".google.protobuf.FieldDescriptorProto.Ty" "pe\022\021\n\ttype_name\030\006 \001(\t\022\020\n\010extendee\030\002 \001(\t\022" "\025\n\rdefault_value\030\007 \001(\t\022\023\n\013oneof_index\030\t " - "\001(\005\022.\n\007options\030\010 \001(\0132\035.google.protobuf.F" - "ieldOptions\"\266\002\n\004Type\022\017\n\013TYPE_DOUBLE\020\001\022\016\n" - "\nTYPE_FLOAT\020\002\022\016\n\nTYPE_INT64\020\003\022\017\n\013TYPE_UI" - "NT64\020\004\022\016\n\nTYPE_INT32\020\005\022\020\n\014TYPE_FIXED64\020\006" - "\022\020\n\014TYPE_FIXED32\020\007\022\r\n\tTYPE_BOOL\020\010\022\017\n\013TYP" - "E_STRING\020\t\022\016\n\nTYPE_GROUP\020\n\022\020\n\014TYPE_MESSA" - "GE\020\013\022\016\n\nTYPE_BYTES\020\014\022\017\n\013TYPE_UINT32\020\r\022\r\n" - "\tTYPE_ENUM\020\016\022\021\n\rTYPE_SFIXED32\020\017\022\021\n\rTYPE_" - "SFIXED64\020\020\022\017\n\013TYPE_SINT32\020\021\022\017\n\013TYPE_SINT" - "64\020\022\"C\n\005Label\022\022\n\016LABEL_OPTIONAL\020\001\022\022\n\016LAB" - "EL_REQUIRED\020\002\022\022\n\016LABEL_REPEATED\020\003\"$\n\024One" - "ofDescriptorProto\022\014\n\004name\030\001 \001(\t\"\214\001\n\023Enum" - "DescriptorProto\022\014\n\004name\030\001 \001(\t\0228\n\005value\030\002" - " \003(\0132).google.protobuf.EnumValueDescript" - "orProto\022-\n\007options\030\003 \001(\0132\034.google.protob" - "uf.EnumOptions\"l\n\030EnumValueDescriptorPro" - "to\022\014\n\004name\030\001 \001(\t\022\016\n\006number\030\002 \001(\005\0222\n\007opti" - "ons\030\003 \001(\0132!.google.protobuf.EnumValueOpt" - "ions\"\220\001\n\026ServiceDescriptorProto\022\014\n\004name\030" - "\001 \001(\t\0226\n\006method\030\002 \003(\0132&.google.protobuf." - "MethodDescriptorProto\0220\n\007options\030\003 \001(\0132\037" - ".google.protobuf.ServiceOptions\"\301\001\n\025Meth" - "odDescriptorProto\022\014\n\004name\030\001 \001(\t\022\022\n\ninput" - "_type\030\002 \001(\t\022\023\n\013output_type\030\003 \001(\t\022/\n\007opti" - "ons\030\004 \001(\0132\036.google.protobuf.MethodOption" - "s\022\037\n\020client_streaming\030\005 \001(\010:\005false\022\037\n\020se" - "rver_streaming\030\006 \001(\010:\005false\"\252\005\n\013FileOpti" - "ons\022\024\n\014java_package\030\001 \001(\t\022\034\n\024java_outer_" - "classname\030\010 \001(\t\022\"\n\023java_multiple_files\030\n" - " \001(\010:\005false\022,\n\035java_generate_equals_and_" - "hash\030\024 \001(\010:\005false\022%\n\026java_string_check_u" - "tf8\030\033 \001(\010:\005false\022F\n\014optimize_for\030\t \001(\0162)" - ".google.protobuf.FileOptions.OptimizeMod" - "e:\005SPEED\022\022\n\ngo_package\030\013 \001(\t\022\"\n\023cc_gener" - "ic_services\030\020 \001(\010:\005false\022$\n\025java_generic" - "_services\030\021 \001(\010:\005false\022\"\n\023py_generic_ser" - "vices\030\022 \001(\010:\005false\022\031\n\ndeprecated\030\027 \001(\010:\005" - "false\022\037\n\020cc_enable_arenas\030\037 \001(\010:\005false\022\031" - "\n\021objc_class_prefix\030$ \001(\t\022\030\n\020csharp_name" - "space\030% \001(\t\022\'\n\037javanano_use_deprecated_p" - "ackage\030& \001(\010\022C\n\024uninterpreted_option\030\347\007 " - "\003(\0132$.google.protobuf.UninterpretedOptio" - "n\":\n\014OptimizeMode\022\t\n\005SPEED\020\001\022\r\n\tCODE_SIZ" - "E\020\002\022\020\n\014LITE_RUNTIME\020\003*\t\010\350\007\020\200\200\200\200\002\"\346\001\n\016Mes" - "sageOptions\022&\n\027message_set_wire_format\030\001" - " \001(\010:\005false\022.\n\037no_standard_descriptor_ac" - "cessor\030\002 \001(\010:\005false\022\031\n\ndeprecated\030\003 \001(\010:" - "\005false\022\021\n\tmap_entry\030\007 \001(\010\022C\n\024uninterpret" - "ed_option\030\347\007 \003(\0132$.google.protobuf.Unint" - "erpretedOption*\t\010\350\007\020\200\200\200\200\002\"\230\003\n\014FieldOptio" - "ns\022:\n\005ctype\030\001 \001(\0162#.google.protobuf.Fiel" - "dOptions.CType:\006STRING\022\016\n\006packed\030\002 \001(\010\022\?" - "\n\006jstype\030\006 \001(\0162$.google.protobuf.FieldOp" - "tions.JSType:\tJS_NORMAL\022\023\n\004lazy\030\005 \001(\010:\005f" - "alse\022\031\n\ndeprecated\030\003 \001(\010:\005false\022\023\n\004weak\030" - "\n \001(\010:\005false\022C\n\024uninterpreted_option\030\347\007 " - "\003(\0132$.google.protobuf.UninterpretedOptio" - "n\"/\n\005CType\022\n\n\006STRING\020\000\022\010\n\004CORD\020\001\022\020\n\014STRI" - "NG_PIECE\020\002\"5\n\006JSType\022\r\n\tJS_NORMAL\020\000\022\r\n\tJ" - "S_STRING\020\001\022\r\n\tJS_NUMBER\020\002*\t\010\350\007\020\200\200\200\200\002\"\215\001\n" - "\013EnumOptions\022\023\n\013allow_alias\030\002 \001(\010\022\031\n\ndep" - "recated\030\003 \001(\010:\005false\022C\n\024uninterpreted_op" - "tion\030\347\007 \003(\0132$.google.protobuf.Uninterpre" - "tedOption*\t\010\350\007\020\200\200\200\200\002\"}\n\020EnumValueOptions" - "\022\031\n\ndeprecated\030\001 \001(\010:\005false\022C\n\024uninterpr" - "eted_option\030\347\007 \003(\0132$.google.protobuf.Uni" - "nterpretedOption*\t\010\350\007\020\200\200\200\200\002\"{\n\016ServiceOp" - "tions\022\031\n\ndeprecated\030! \001(\010:\005false\022C\n\024unin" - "terpreted_option\030\347\007 \003(\0132$.google.protobu" - "f.UninterpretedOption*\t\010\350\007\020\200\200\200\200\002\"z\n\rMeth" - "odOptions\022\031\n\ndeprecated\030! \001(\010:\005false\022C\n\024" - "uninterpreted_option\030\347\007 \003(\0132$.google.pro" - "tobuf.UninterpretedOption*\t\010\350\007\020\200\200\200\200\002\"\236\002\n" - "\023UninterpretedOption\022;\n\004name\030\002 \003(\0132-.goo" - "gle.protobuf.UninterpretedOption.NamePar" - "t\022\030\n\020identifier_value\030\003 \001(\t\022\032\n\022positive_" - "int_value\030\004 \001(\004\022\032\n\022negative_int_value\030\005 " - "\001(\003\022\024\n\014double_value\030\006 \001(\001\022\024\n\014string_valu" - "e\030\007 \001(\014\022\027\n\017aggregate_value\030\010 \001(\t\0323\n\010Name" - "Part\022\021\n\tname_part\030\001 \002(\t\022\024\n\014is_extension\030" - "\002 \002(\010\"\325\001\n\016SourceCodeInfo\022:\n\010location\030\001 \003" - "(\0132(.google.protobuf.SourceCodeInfo.Loca" - "tion\032\206\001\n\010Location\022\020\n\004path\030\001 \003(\005B\002\020\001\022\020\n\004s" - "pan\030\002 \003(\005B\002\020\001\022\030\n\020leading_comments\030\003 \001(\t\022" - "\031\n\021trailing_comments\030\004 \001(\t\022!\n\031leading_de" - "tached_comments\030\006 \003(\tB;\n\023com.google.prot" - "obufB\020DescriptorProtosH\001Z\ndescriptor\242\002\003G" - "PB", 4962); + "\001(\005\022\021\n\tjson_name\030\n \001(\t\022.\n\007options\030\010 \001(\0132" + "\035.google.protobuf.FieldOptions\"\266\002\n\004Type\022" + "\017\n\013TYPE_DOUBLE\020\001\022\016\n\nTYPE_FLOAT\020\002\022\016\n\nTYPE" + "_INT64\020\003\022\017\n\013TYPE_UINT64\020\004\022\016\n\nTYPE_INT32\020" + "\005\022\020\n\014TYPE_FIXED64\020\006\022\020\n\014TYPE_FIXED32\020\007\022\r\n" + "\tTYPE_BOOL\020\010\022\017\n\013TYPE_STRING\020\t\022\016\n\nTYPE_GR" + "OUP\020\n\022\020\n\014TYPE_MESSAGE\020\013\022\016\n\nTYPE_BYTES\020\014\022" + "\017\n\013TYPE_UINT32\020\r\022\r\n\tTYPE_ENUM\020\016\022\021\n\rTYPE_" + "SFIXED32\020\017\022\021\n\rTYPE_SFIXED64\020\020\022\017\n\013TYPE_SI" + "NT32\020\021\022\017\n\013TYPE_SINT64\020\022\"C\n\005Label\022\022\n\016LABE" + "L_OPTIONAL\020\001\022\022\n\016LABEL_REQUIRED\020\002\022\022\n\016LABE" + "L_REPEATED\020\003\"$\n\024OneofDescriptorProto\022\014\n\004" + "name\030\001 \001(\t\"\214\001\n\023EnumDescriptorProto\022\014\n\004na" + "me\030\001 \001(\t\0228\n\005value\030\002 \003(\0132).google.protobu" + "f.EnumValueDescriptorProto\022-\n\007options\030\003 " + "\001(\0132\034.google.protobuf.EnumOptions\"l\n\030Enu" + "mValueDescriptorProto\022\014\n\004name\030\001 \001(\t\022\016\n\006n" + "umber\030\002 \001(\005\0222\n\007options\030\003 \001(\0132!.google.pr" + "otobuf.EnumValueOptions\"\220\001\n\026ServiceDescr" + "iptorProto\022\014\n\004name\030\001 \001(\t\0226\n\006method\030\002 \003(\013" + "2&.google.protobuf.MethodDescriptorProto" + "\0220\n\007options\030\003 \001(\0132\037.google.protobuf.Serv" + "iceOptions\"\301\001\n\025MethodDescriptorProto\022\014\n\004" + "name\030\001 \001(\t\022\022\n\ninput_type\030\002 \001(\t\022\023\n\013output" + "_type\030\003 \001(\t\022/\n\007options\030\004 \001(\0132\036.google.pr" + "otobuf.MethodOptions\022\037\n\020client_streaming" + "\030\005 \001(\010:\005false\022\037\n\020server_streaming\030\006 \001(\010:" + "\005false\"\252\005\n\013FileOptions\022\024\n\014java_package\030\001" + " \001(\t\022\034\n\024java_outer_classname\030\010 \001(\t\022\"\n\023ja" + "va_multiple_files\030\n \001(\010:\005false\022,\n\035java_g" + "enerate_equals_and_hash\030\024 \001(\010:\005false\022%\n\026" + "java_string_check_utf8\030\033 \001(\010:\005false\022F\n\014o" + "ptimize_for\030\t \001(\0162).google.protobuf.File" + "Options.OptimizeMode:\005SPEED\022\022\n\ngo_packag" + "e\030\013 \001(\t\022\"\n\023cc_generic_services\030\020 \001(\010:\005fa" + "lse\022$\n\025java_generic_services\030\021 \001(\010:\005fals" + "e\022\"\n\023py_generic_services\030\022 \001(\010:\005false\022\031\n" + "\ndeprecated\030\027 \001(\010:\005false\022\037\n\020cc_enable_ar" + "enas\030\037 \001(\010:\005false\022\031\n\021objc_class_prefix\030$" + " \001(\t\022\030\n\020csharp_namespace\030% \001(\t\022\'\n\037javana" + "no_use_deprecated_package\030& \001(\010\022C\n\024unint" + "erpreted_option\030\347\007 \003(\0132$.google.protobuf" + ".UninterpretedOption\":\n\014OptimizeMode\022\t\n\005" + "SPEED\020\001\022\r\n\tCODE_SIZE\020\002\022\020\n\014LITE_RUNTIME\020\003" + "*\t\010\350\007\020\200\200\200\200\002\"\346\001\n\016MessageOptions\022&\n\027messag" + "e_set_wire_format\030\001 \001(\010:\005false\022.\n\037no_sta" + "ndard_descriptor_accessor\030\002 \001(\010:\005false\022\031" + "\n\ndeprecated\030\003 \001(\010:\005false\022\021\n\tmap_entry\030\007" + " \001(\010\022C\n\024uninterpreted_option\030\347\007 \003(\0132$.go" + "ogle.protobuf.UninterpretedOption*\t\010\350\007\020\200" + "\200\200\200\002\"\230\003\n\014FieldOptions\022:\n\005ctype\030\001 \001(\0162#.g" + "oogle.protobuf.FieldOptions.CType:\006STRIN" + "G\022\016\n\006packed\030\002 \001(\010\022\?\n\006jstype\030\006 \001(\0162$.goog" + "le.protobuf.FieldOptions.JSType:\tJS_NORM" + "AL\022\023\n\004lazy\030\005 \001(\010:\005false\022\031\n\ndeprecated\030\003 " + "\001(\010:\005false\022\023\n\004weak\030\n \001(\010:\005false\022C\n\024unint" + "erpreted_option\030\347\007 \003(\0132$.google.protobuf" + ".UninterpretedOption\"/\n\005CType\022\n\n\006STRING\020" + "\000\022\010\n\004CORD\020\001\022\020\n\014STRING_PIECE\020\002\"5\n\006JSType\022" + "\r\n\tJS_NORMAL\020\000\022\r\n\tJS_STRING\020\001\022\r\n\tJS_NUMB" + "ER\020\002*\t\010\350\007\020\200\200\200\200\002\"\215\001\n\013EnumOptions\022\023\n\013allow" + "_alias\030\002 \001(\010\022\031\n\ndeprecated\030\003 \001(\010:\005false\022" + "C\n\024uninterpreted_option\030\347\007 \003(\0132$.google." + "protobuf.UninterpretedOption*\t\010\350\007\020\200\200\200\200\002\"" + "}\n\020EnumValueOptions\022\031\n\ndeprecated\030\001 \001(\010:" + "\005false\022C\n\024uninterpreted_option\030\347\007 \003(\0132$." + "google.protobuf.UninterpretedOption*\t\010\350\007" + "\020\200\200\200\200\002\"{\n\016ServiceOptions\022\031\n\ndeprecated\030!" + " \001(\010:\005false\022C\n\024uninterpreted_option\030\347\007 \003" + "(\0132$.google.protobuf.UninterpretedOption" + "*\t\010\350\007\020\200\200\200\200\002\"z\n\rMethodOptions\022\031\n\ndeprecat" + "ed\030! \001(\010:\005false\022C\n\024uninterpreted_option\030" + "\347\007 \003(\0132$.google.protobuf.UninterpretedOp" + "tion*\t\010\350\007\020\200\200\200\200\002\"\236\002\n\023UninterpretedOption\022" + ";\n\004name\030\002 \003(\0132-.google.protobuf.Uninterp" + "retedOption.NamePart\022\030\n\020identifier_value" + "\030\003 \001(\t\022\032\n\022positive_int_value\030\004 \001(\004\022\032\n\022ne" + "gative_int_value\030\005 \001(\003\022\024\n\014double_value\030\006" + " \001(\001\022\024\n\014string_value\030\007 \001(\014\022\027\n\017aggregate_" + "value\030\010 \001(\t\0323\n\010NamePart\022\021\n\tname_part\030\001 \002" + "(\t\022\024\n\014is_extension\030\002 \002(\010\"\325\001\n\016SourceCodeI" + "nfo\022:\n\010location\030\001 \003(\0132(.google.protobuf." + "SourceCodeInfo.Location\032\206\001\n\010Location\022\020\n\004" + "path\030\001 \003(\005B\002\020\001\022\020\n\004span\030\002 \003(\005B\002\020\001\022\030\n\020lead" + "ing_comments\030\003 \001(\t\022\031\n\021trailing_comments\030" + "\004 \001(\t\022!\n\031leading_detached_comments\030\006 \003(\t" + "B;\n\023com.google.protobufB\020DescriptorProto" + "sH\001Z\ndescriptor\242\002\003GPB", 4981); ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile( "google/protobuf/descriptor.proto", &protobuf_RegisterTypes); FileDescriptorSet::default_instance_ = new FileDescriptorSet(); @@ -4109,6 +4110,7 @@ const int FieldDescriptorProto::kTypeNameFieldNumber; const int FieldDescriptorProto::kExtendeeFieldNumber; const int FieldDescriptorProto::kDefaultValueFieldNumber; const int FieldDescriptorProto::kOneofIndexFieldNumber; +const int FieldDescriptorProto::kJsonNameFieldNumber; const int FieldDescriptorProto::kOptionsFieldNumber; #endif // !_MSC_VER @@ -4141,6 +4143,7 @@ void FieldDescriptorProto::SharedCtor() { extendee_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); default_value_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); oneof_index_ = 0; + json_name_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); options_ = NULL; ::memset(_has_bits_, 0, sizeof(_has_bits_)); } @@ -4155,6 +4158,7 @@ void FieldDescriptorProto::SharedDtor() { type_name_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); extendee_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); default_value_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + json_name_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); if (this != default_instance_) { delete options_; } @@ -4204,8 +4208,13 @@ void FieldDescriptorProto::Clear() { } oneof_index_ = 0; } - if (has_options()) { - if (options_ != NULL) options_->::google::protobuf::FieldOptions::Clear(); + if (_has_bits_[8 / 32] & 768u) { + if (has_json_name()) { + json_name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + } + if (has_options()) { + if (options_ != NULL) options_->::google::protobuf::FieldOptions::Clear(); + } } ::memset(_has_bits_, 0, sizeof(_has_bits_)); if (_internal_metadata_.have_unknown_fields()) { @@ -4369,6 +4378,23 @@ bool FieldDescriptorProto::MergePartialFromCodedStream( } else { goto handle_unusual; } + if (input->ExpectTag(82)) goto parse_json_name; + break; + } + + // optional string json_name = 10; + case 10: { + if (tag == 82) { + parse_json_name: + DO_(::google::protobuf::internal::WireFormatLite::ReadString( + input, this->mutable_json_name())); + ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( + this->json_name().data(), this->json_name().length(), + ::google::protobuf::internal::WireFormat::PARSE, + "google.protobuf.FieldDescriptorProto.json_name"); + } else { + goto handle_unusual; + } if (input->ExpectAtEnd()) goto success; break; } @@ -4466,6 +4492,16 @@ void FieldDescriptorProto::SerializeWithCachedSizes( ::google::protobuf::internal::WireFormatLite::WriteInt32(9, this->oneof_index(), output); } + // optional string json_name = 10; + if (has_json_name()) { + ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( + this->json_name().data(), this->json_name().length(), + ::google::protobuf::internal::WireFormat::SERIALIZE, + "google.protobuf.FieldDescriptorProto.json_name"); + ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( + 10, this->json_name(), output); + } + if (_internal_metadata_.have_unknown_fields()) { ::google::protobuf::internal::WireFormat::SerializeUnknownFields( unknown_fields(), output); @@ -4549,6 +4585,17 @@ void FieldDescriptorProto::SerializeWithCachedSizes( target = ::google::protobuf::internal::WireFormatLite::WriteInt32ToArray(9, this->oneof_index(), target); } + // optional string json_name = 10; + if (has_json_name()) { + ::google::protobuf::internal::WireFormat::VerifyUTF8StringNamedField( + this->json_name().data(), this->json_name().length(), + ::google::protobuf::internal::WireFormat::SERIALIZE, + "google.protobuf.FieldDescriptorProto.json_name"); + target = + ::google::protobuf::internal::WireFormatLite::WriteStringToArray( + 10, this->json_name(), target); + } + if (_internal_metadata_.have_unknown_fields()) { target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray( unknown_fields(), target); @@ -4616,13 +4663,22 @@ int FieldDescriptorProto::ByteSize() const { } } - // optional .google.protobuf.FieldOptions options = 8; - if (has_options()) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( - *this->options_); - } + if (_has_bits_[8 / 32] & 768u) { + // optional string json_name = 10; + if (has_json_name()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::StringSize( + this->json_name()); + } + // optional .google.protobuf.FieldOptions options = 8; + if (has_options()) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( + *this->options_); + } + + } if (_internal_metadata_.have_unknown_fields()) { total_size += ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize( @@ -4679,6 +4735,10 @@ void FieldDescriptorProto::MergeFrom(const FieldDescriptorProto& from) { } } if (from._has_bits_[8 / 32] & (0xffu << (8 % 32))) { + if (from.has_json_name()) { + set_has_json_name(); + json_name_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.json_name_); + } if (from.has_options()) { mutable_options()->::google::protobuf::FieldOptions::MergeFrom(from.options()); } @@ -4721,6 +4781,7 @@ void FieldDescriptorProto::InternalSwap(FieldDescriptorProto* other) { extendee_.Swap(&other->extendee_); default_value_.Swap(&other->default_value_); std::swap(oneof_index_, other->oneof_index_); + json_name_.Swap(&other->json_name_); std::swap(options_, other->options_); std::swap(_has_bits_[0], other->_has_bits_[0]); _internal_metadata_.Swap(&other->_internal_metadata_); @@ -5048,15 +5109,68 @@ void FieldDescriptorProto::clear_oneof_index() { // @@protoc_insertion_point(field_set:google.protobuf.FieldDescriptorProto.oneof_index) } +// optional string json_name = 10; +bool FieldDescriptorProto::has_json_name() const { + return (_has_bits_[0] & 0x00000100u) != 0; +} +void FieldDescriptorProto::set_has_json_name() { + _has_bits_[0] |= 0x00000100u; +} +void FieldDescriptorProto::clear_has_json_name() { + _has_bits_[0] &= ~0x00000100u; +} +void FieldDescriptorProto::clear_json_name() { + json_name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + clear_has_json_name(); +} + const ::std::string& FieldDescriptorProto::json_name() const { + // @@protoc_insertion_point(field_get:google.protobuf.FieldDescriptorProto.json_name) + return json_name_.GetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} + void FieldDescriptorProto::set_json_name(const ::std::string& value) { + set_has_json_name(); + json_name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:google.protobuf.FieldDescriptorProto.json_name) +} + void FieldDescriptorProto::set_json_name(const char* value) { + set_has_json_name(); + json_name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:google.protobuf.FieldDescriptorProto.json_name) +} + void FieldDescriptorProto::set_json_name(const char* value, size_t size) { + set_has_json_name(); + json_name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:google.protobuf.FieldDescriptorProto.json_name) +} + ::std::string* FieldDescriptorProto::mutable_json_name() { + set_has_json_name(); + // @@protoc_insertion_point(field_mutable:google.protobuf.FieldDescriptorProto.json_name) + return json_name_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} + ::std::string* FieldDescriptorProto::release_json_name() { + clear_has_json_name(); + return json_name_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} + void FieldDescriptorProto::set_allocated_json_name(::std::string* json_name) { + if (json_name != NULL) { + set_has_json_name(); + } else { + clear_has_json_name(); + } + json_name_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), json_name); + // @@protoc_insertion_point(field_set_allocated:google.protobuf.FieldDescriptorProto.json_name) +} + // optional .google.protobuf.FieldOptions options = 8; bool FieldDescriptorProto::has_options() const { - return (_has_bits_[0] & 0x00000100u) != 0; + return (_has_bits_[0] & 0x00000200u) != 0; } void FieldDescriptorProto::set_has_options() { - _has_bits_[0] |= 0x00000100u; + _has_bits_[0] |= 0x00000200u; } void FieldDescriptorProto::clear_has_options() { - _has_bits_[0] &= ~0x00000100u; + _has_bits_[0] &= ~0x00000200u; } void FieldDescriptorProto::clear_options() { if (options_ != NULL) options_->::google::protobuf::FieldOptions::Clear(); diff --git a/src/google/protobuf/descriptor.pb.h b/src/google/protobuf/descriptor.pb.h index 931ff02d61..6025516258 100644 --- a/src/google/protobuf/descriptor.pb.h +++ b/src/google/protobuf/descriptor.pb.h @@ -1133,6 +1133,18 @@ class LIBPROTOBUF_EXPORT FieldDescriptorProto : public ::google::protobuf::Messa ::google::protobuf::int32 oneof_index() const; void set_oneof_index(::google::protobuf::int32 value); + // optional string json_name = 10; + bool has_json_name() const; + void clear_json_name(); + static const int kJsonNameFieldNumber = 10; + const ::std::string& json_name() const; + void set_json_name(const ::std::string& value); + void set_json_name(const char* value); + void set_json_name(const char* value, size_t size); + ::std::string* mutable_json_name(); + ::std::string* release_json_name(); + void set_allocated_json_name(::std::string* json_name); + // optional .google.protobuf.FieldOptions options = 8; bool has_options() const; void clear_options(); @@ -1160,6 +1172,8 @@ class LIBPROTOBUF_EXPORT FieldDescriptorProto : public ::google::protobuf::Messa inline void clear_has_default_value(); inline void set_has_oneof_index(); inline void clear_has_oneof_index(); + inline void set_has_json_name(); + inline void clear_has_json_name(); inline void set_has_options(); inline void clear_has_options(); @@ -1174,6 +1188,7 @@ class LIBPROTOBUF_EXPORT FieldDescriptorProto : public ::google::protobuf::Messa int type_; ::google::protobuf::int32 oneof_index_; ::google::protobuf::internal::ArenaStringPtr default_value_; + ::google::protobuf::internal::ArenaStringPtr json_name_; ::google::protobuf::FieldOptions* options_; friend void LIBPROTOBUF_EXPORT protobuf_AddDesc_google_2fprotobuf_2fdescriptor_2eproto(); friend void protobuf_AssignDesc_google_2fprotobuf_2fdescriptor_2eproto(); @@ -4678,15 +4693,68 @@ inline void FieldDescriptorProto::set_oneof_index(::google::protobuf::int32 valu // @@protoc_insertion_point(field_set:google.protobuf.FieldDescriptorProto.oneof_index) } +// optional string json_name = 10; +inline bool FieldDescriptorProto::has_json_name() const { + return (_has_bits_[0] & 0x00000100u) != 0; +} +inline void FieldDescriptorProto::set_has_json_name() { + _has_bits_[0] |= 0x00000100u; +} +inline void FieldDescriptorProto::clear_has_json_name() { + _has_bits_[0] &= ~0x00000100u; +} +inline void FieldDescriptorProto::clear_json_name() { + json_name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + clear_has_json_name(); +} +inline const ::std::string& FieldDescriptorProto::json_name() const { + // @@protoc_insertion_point(field_get:google.protobuf.FieldDescriptorProto.json_name) + return json_name_.GetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline void FieldDescriptorProto::set_json_name(const ::std::string& value) { + set_has_json_name(); + json_name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:google.protobuf.FieldDescriptorProto.json_name) +} +inline void FieldDescriptorProto::set_json_name(const char* value) { + set_has_json_name(); + json_name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:google.protobuf.FieldDescriptorProto.json_name) +} +inline void FieldDescriptorProto::set_json_name(const char* value, size_t size) { + set_has_json_name(); + json_name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:google.protobuf.FieldDescriptorProto.json_name) +} +inline ::std::string* FieldDescriptorProto::mutable_json_name() { + set_has_json_name(); + // @@protoc_insertion_point(field_mutable:google.protobuf.FieldDescriptorProto.json_name) + return json_name_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline ::std::string* FieldDescriptorProto::release_json_name() { + clear_has_json_name(); + return json_name_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline void FieldDescriptorProto::set_allocated_json_name(::std::string* json_name) { + if (json_name != NULL) { + set_has_json_name(); + } else { + clear_has_json_name(); + } + json_name_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), json_name); + // @@protoc_insertion_point(field_set_allocated:google.protobuf.FieldDescriptorProto.json_name) +} + // optional .google.protobuf.FieldOptions options = 8; inline bool FieldDescriptorProto::has_options() const { - return (_has_bits_[0] & 0x00000100u) != 0; + return (_has_bits_[0] & 0x00000200u) != 0; } inline void FieldDescriptorProto::set_has_options() { - _has_bits_[0] |= 0x00000100u; + _has_bits_[0] |= 0x00000200u; } inline void FieldDescriptorProto::clear_has_options() { - _has_bits_[0] &= ~0x00000100u; + _has_bits_[0] &= ~0x00000200u; } inline void FieldDescriptorProto::clear_options() { if (options_ != NULL) options_->::google::protobuf::FieldOptions::Clear(); diff --git a/src/google/protobuf/descriptor.proto b/src/google/protobuf/descriptor.proto index 8f90a956cf..801d85e446 100644 --- a/src/google/protobuf/descriptor.proto +++ b/src/google/protobuf/descriptor.proto @@ -191,6 +191,12 @@ message FieldDescriptorProto { // list. This field is a member of that oneof. optional int32 oneof_index = 9; + // JSON name of this field. The value is set by protocol compiler. If the + // user has set a "json_name" option on this field, that option's value + // will be used. Otherwise, it's deduced from the field's name by converting + // it to camelCase. + optional string json_name = 10; + optional FieldOptions options = 8; } diff --git a/src/google/protobuf/descriptor_database_unittest.cc b/src/google/protobuf/descriptor_database_unittest.cc index a87fa04915..1fc3816e16 100644 --- a/src/google/protobuf/descriptor_database_unittest.cc +++ b/src/google/protobuf/descriptor_database_unittest.cc @@ -35,6 +35,10 @@ // This file makes extensive use of RFC 3092. :) #include +#include +#ifndef _SHARED_PTR_H +#include +#endif #include #include @@ -177,7 +181,7 @@ class DescriptorDatabaseTest EXPECT_FALSE(test_case_->AddToDatabase(file_proto)); } - scoped_ptr test_case_; + google::protobuf::scoped_ptr test_case_; DescriptorDatabase* database_; }; diff --git a/src/google/protobuf/descriptor_unittest.cc b/src/google/protobuf/descriptor_unittest.cc index e9b027dbb6..ccd0650dc3 100644 --- a/src/google/protobuf/descriptor_unittest.cc +++ b/src/google/protobuf/descriptor_unittest.cc @@ -34,6 +34,10 @@ // // This file makes extensive use of RFC 3092. :) +#include +#ifndef _SHARED_PTR_H +#include +#endif #include #include @@ -461,6 +465,16 @@ class DescriptorTest : public testing::Test { // map map_int32_int32 = 1; // } // + // // in "json.proto" + // message TestMessage4 { + // optional int32 field_name1 = 1; + // optional int32 fieldName2 = 2; + // optional int32 FieldName3 = 3; + // optional int32 _field_name4 = 4; + // optional int32 FIELD_NAME5 = 5; + // optional int32 field_name6 = 6 [json_name = "@type"]; + // } + // // We cheat and use TestForeign as the type for qux rather than create // an actual nested type. // @@ -526,6 +540,30 @@ class DescriptorTest : public testing::Test { FieldDescriptorProto::TYPE_MESSAGE) ->set_type_name("MapInt32Int32Entry"); + FileDescriptorProto json_file; + json_file.set_name("json.proto"); + json_file.set_syntax("proto3"); + DescriptorProto* message4 = AddMessage(&json_file, "TestMessage4"); + AddField(message4, "field_name1", 1, + FieldDescriptorProto::LABEL_OPTIONAL, + FieldDescriptorProto::TYPE_INT32); + AddField(message4, "fieldName2", 2, + FieldDescriptorProto::LABEL_OPTIONAL, + FieldDescriptorProto::TYPE_INT32); + AddField(message4, "FieldName3", 3, + FieldDescriptorProto::LABEL_OPTIONAL, + FieldDescriptorProto::TYPE_INT32); + AddField(message4, "_field_name4", 4, + FieldDescriptorProto::LABEL_OPTIONAL, + FieldDescriptorProto::TYPE_INT32); + AddField(message4, "FIELD_NAME5", 5, + FieldDescriptorProto::LABEL_OPTIONAL, + FieldDescriptorProto::TYPE_INT32); + AddField(message4, "field_name6", 6, + FieldDescriptorProto::LABEL_OPTIONAL, + FieldDescriptorProto::TYPE_INT32) + ->set_json_name("@type"); + // Build the descriptors and get the pointers. foo_file_ = pool_.BuildFile(foo_file); ASSERT_TRUE(foo_file_ != NULL); @@ -536,6 +574,9 @@ class DescriptorTest : public testing::Test { map_file_ = pool_.BuildFile(map_file); ASSERT_TRUE(map_file_ != NULL); + json_file_ = pool_.BuildFile(json_file); + ASSERT_TRUE(json_file_ != NULL); + ASSERT_EQ(1, foo_file_->enum_type_count()); enum_ = foo_file_->enum_type(0); @@ -562,6 +603,14 @@ class DescriptorTest : public testing::Test { ASSERT_EQ(1, message3_->field_count()); map_ = message3_->field(0); + + ASSERT_EQ(1, json_file_->message_type_count()); + message4_ = json_file_->message_type(0); + } + + void CopyWithJsonName(const Descriptor* message, DescriptorProto* proto) { + message->CopyTo(proto); + message->CopyJsonNameTo(proto); } DescriptorPool pool_; @@ -569,10 +618,12 @@ class DescriptorTest : public testing::Test { const FileDescriptor* foo_file_; const FileDescriptor* bar_file_; const FileDescriptor* map_file_; + const FileDescriptor* json_file_; const Descriptor* message_; const Descriptor* message2_; const Descriptor* message3_; + const Descriptor* message4_; const Descriptor* foreign_; const EnumDescriptor* enum_; @@ -664,6 +715,35 @@ TEST_F(DescriptorTest, FieldFullName) { EXPECT_EQ("corge.grault.TestMessage2.quux", quux2_->full_name()); } +TEST_F(DescriptorTest, FieldJsonName) { + EXPECT_EQ("fieldName1", message4_->field(0)->json_name()); + EXPECT_EQ("fieldName2", message4_->field(1)->json_name()); + EXPECT_EQ("fieldName3", message4_->field(2)->json_name()); + EXPECT_EQ("fieldName4", message4_->field(3)->json_name()); + EXPECT_EQ("fIELDNAME5", message4_->field(4)->json_name()); + EXPECT_EQ("@type", message4_->field(5)->json_name()); + + DescriptorProto proto; + message4_->CopyTo(&proto); + ASSERT_EQ(6, proto.field_size()); + EXPECT_FALSE(proto.field(0).has_json_name()); + EXPECT_FALSE(proto.field(1).has_json_name()); + EXPECT_FALSE(proto.field(2).has_json_name()); + EXPECT_FALSE(proto.field(3).has_json_name()); + EXPECT_FALSE(proto.field(4).has_json_name()); + EXPECT_EQ("@type", proto.field(5).json_name()); + + proto.Clear(); + CopyWithJsonName(message4_, &proto); + ASSERT_EQ(6, proto.field_size()); + EXPECT_EQ("fieldName1", proto.field(0).json_name()); + EXPECT_EQ("fieldName2", proto.field(1).json_name()); + EXPECT_EQ("fieldName3", proto.field(2).json_name()); + EXPECT_EQ("fieldName4", proto.field(3).json_name()); + EXPECT_EQ("fIELDNAME5", proto.field(4).json_name()); + EXPECT_EQ("@type", proto.field(5).json_name()); +} + TEST_F(DescriptorTest, FieldFile) { EXPECT_EQ(foo_file_, foo_->file()); EXPECT_EQ(foo_file_, bar_->file()); @@ -1900,7 +1980,7 @@ class MiscTest : public testing::Test { return field != NULL ? field->enum_type() : NULL; } - scoped_ptr pool_; + google::protobuf::scoped_ptr pool_; }; TEST_F(MiscTest, TypeNames) { @@ -2330,7 +2410,7 @@ class AllowUnknownDependenciesTest const FieldDescriptor* qux_field_; SimpleDescriptorDatabase db_; // used if in FALLBACK_DATABASE mode. - scoped_ptr pool_; + google::protobuf::scoped_ptr pool_; }; TEST_P(AllowUnknownDependenciesTest, PlaceholderFile) { @@ -5681,6 +5761,32 @@ TEST_F(ValidationErrorTest, ValidateProto3Extension) { "defining options.\n"); } +// Test that field names that may conflict in JSON is not allowed by protoc. +TEST_F(ValidationErrorTest, ValidateProto3JsonName) { + // The comparison is case-insensitive. + BuildFileWithErrors( + "name: 'foo.proto' " + "syntax: 'proto3' " + "message_type {" + " name: 'Foo'" + " field { name:'name' number:1 label:LABEL_OPTIONAL type:TYPE_INT32 }" + " field { name:'Name' number:2 label:LABEL_OPTIONAL type:TYPE_INT32 }" + "}", + "foo.proto: Foo: OTHER: The JSON camcel-case name of field \"Name\" " + "conflicts with field \"name\". This is not allowed in proto3.\n"); + // Underscores are ignored. + BuildFileWithErrors( + "name: 'foo.proto' " + "syntax: 'proto3' " + "message_type {" + " name: 'Foo'" + " field { name:'ab' number:1 label:LABEL_OPTIONAL type:TYPE_INT32 }" + " field { name:'_a__b_' number:2 label:LABEL_OPTIONAL type:TYPE_INT32 }" + "}", + "foo.proto: Foo: OTHER: The JSON camcel-case name of field \"_a__b_\" " + "conflicts with field \"ab\". This is not allowed in proto3.\n"); +} + // =================================================================== // DescriptorDatabase diff --git a/src/google/protobuf/dynamic_message.cc b/src/google/protobuf/dynamic_message.cc index 2324d9525d..091fc97544 100644 --- a/src/google/protobuf/dynamic_message.cc +++ b/src/google/protobuf/dynamic_message.cc @@ -64,6 +64,10 @@ #include #include +#include +#ifndef _SHARED_PTR_H +#include +#endif #include #include @@ -229,8 +233,8 @@ class DynamicMessage : public Message { // Warning: The order in which the following pointers are defined is // important (the prototype must be deleted *before* the offsets). - scoped_array offsets; - scoped_ptr reflection; + google::protobuf::scoped_array offsets; + google::protobuf::scoped_ptr reflection; // Don't use a scoped_ptr to hold the prototype: the destructor for // DynamicMessage needs to know whether it is the prototype, and does so by // looking back at this field. This would assume details about the diff --git a/src/google/protobuf/dynamic_message_unittest.cc b/src/google/protobuf/dynamic_message_unittest.cc index b9796c764f..70e437d7d2 100644 --- a/src/google/protobuf/dynamic_message_unittest.cc +++ b/src/google/protobuf/dynamic_message_unittest.cc @@ -40,6 +40,11 @@ // reflection_ops_unittest, cover the rest of the functionality used by // DynamicMessage. +#include +#ifndef _SHARED_PTR_H +#include +#endif + #include #include #include @@ -144,7 +149,7 @@ TEST_F(DynamicMessageTest, IndependentOffsets) { // Check that all fields have independent offsets by setting each // one to a unique value then checking that they all still have those // unique values (i.e. they don't stomp each other). - scoped_ptr message(prototype_->New()); + google::protobuf::scoped_ptr message(prototype_->New()); TestUtil::ReflectionTester reflection_tester(descriptor_); reflection_tester.SetAllFieldsViaReflection(message.get()); @@ -153,7 +158,7 @@ TEST_F(DynamicMessageTest, IndependentOffsets) { TEST_F(DynamicMessageTest, Extensions) { // Check that extensions work. - scoped_ptr message(extensions_prototype_->New()); + google::protobuf::scoped_ptr message(extensions_prototype_->New()); TestUtil::ReflectionTester reflection_tester(extensions_descriptor_); reflection_tester.SetAllFieldsViaReflection(message.get()); @@ -162,7 +167,7 @@ TEST_F(DynamicMessageTest, Extensions) { TEST_F(DynamicMessageTest, PackedFields) { // Check that packed fields work properly. - scoped_ptr message(packed_prototype_->New()); + google::protobuf::scoped_ptr message(packed_prototype_->New()); TestUtil::ReflectionTester reflection_tester(packed_descriptor_); reflection_tester.SetPackedFieldsViaReflection(message.get()); @@ -171,7 +176,7 @@ TEST_F(DynamicMessageTest, PackedFields) { TEST_F(DynamicMessageTest, Oneof) { // Check that oneof fields work properly. - scoped_ptr message(oneof_prototype_->New()); + google::protobuf::scoped_ptr message(oneof_prototype_->New()); // Check default values. const Descriptor* descriptor = message->GetDescriptor(); @@ -232,7 +237,7 @@ TEST_F(DynamicMessageTest, SpaceUsed) { // Since we share the implementation with generated messages, we don't need // to test very much here. Just make sure it appears to be working. - scoped_ptr message(prototype_->New()); + google::protobuf::scoped_ptr message(prototype_->New()); TestUtil::ReflectionTester reflection_tester(descriptor_); int initial_space_used = message->SpaceUsed(); diff --git a/src/google/protobuf/extension_set.cc b/src/google/protobuf/extension_set.cc index 919bd83b2d..9afb236147 100644 --- a/src/google/protobuf/extension_set.cc +++ b/src/google/protobuf/extension_set.cc @@ -34,6 +34,7 @@ #include #include +#include #include #include #include @@ -594,20 +595,21 @@ void ExtensionSet::SetAllocatedMessage(int number, FieldType type, ClearExtension(number); return; } + ::google::protobuf::Arena* message_arena = message->GetArena(); Extension* extension; if (MaybeNewExtension(number, descriptor, &extension)) { extension->type = type; GOOGLE_DCHECK_EQ(cpp_type(extension->type), WireFormatLite::CPPTYPE_MESSAGE); extension->is_repeated = false; extension->is_lazy = false; - if (message->GetArena() == arena_) { + if (message_arena == arena_) { extension->message_value = message; + } else if (message_arena == NULL) { + extension->message_value = message; + arena_->Own(message); // not NULL because not equal to message_arena } else { extension->message_value = message->New(arena_); extension->message_value->CheckTypeAndMergeFrom(*message); - if (message->GetArena() == NULL) { - delete message; - } } } else { GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE); @@ -617,14 +619,14 @@ void ExtensionSet::SetAllocatedMessage(int number, FieldType type, if (arena_ == NULL) { delete extension->message_value; } - if (message->GetArena() == arena_) { + if (message_arena == arena_) { extension->message_value = message; + } else if (message_arena == NULL) { + extension->message_value = message; + arena_->Own(message); // not NULL because not equal to message_arena } else { extension->message_value = message->New(arena_); extension->message_value->CheckTypeAndMergeFrom(*message); - if (message->GetArena() == NULL) { - delete message; - } } } } diff --git a/src/google/protobuf/extension_set.h b/src/google/protobuf/extension_set.h index 0968125943..40704608d4 100644 --- a/src/google/protobuf/extension_set.h +++ b/src/google/protobuf/extension_set.h @@ -724,8 +724,7 @@ class RepeatedPrimitiveTypeTraits { static const RepeatedFieldType* GetDefaultRepeatedField(); }; -LIBPROTOBUF_EXPORT extern ProtobufOnceType -repeated_primitive_generic_type_traits_once_init_; +extern ProtobufOnceType repeated_primitive_generic_type_traits_once_init_; class LIBPROTOBUF_EXPORT RepeatedPrimitiveGenericTypeTraits { private: @@ -766,7 +765,7 @@ template<> inline void RepeatedPrimitiveTypeTraits::Add( \ } \ template<> inline const RepeatedField* \ RepeatedPrimitiveTypeTraits::GetDefaultRepeatedField() { \ - GoogleOnceInit( \ + ::google::protobuf::GoogleOnceInit( \ &repeated_primitive_generic_type_traits_once_init_, \ &RepeatedPrimitiveGenericTypeTraits::InitializeDefaultRepeatedFields); \ return RepeatedPrimitiveGenericTypeTraits:: \ @@ -822,8 +821,7 @@ class LIBPROTOBUF_EXPORT StringTypeTraits { } }; -LIBPROTOBUF_EXPORT extern ProtobufOnceType -repeated_string_type_traits_once_init_; +extern ProtobufOnceType repeated_string_type_traits_once_init_; class LIBPROTOBUF_EXPORT RepeatedStringTypeTraits { public: @@ -868,7 +866,7 @@ class LIBPROTOBUF_EXPORT RepeatedStringTypeTraits { } static const RepeatedFieldType* GetDefaultRepeatedField() { - GoogleOnceInit(&repeated_string_type_traits_once_init_, + ::google::protobuf::GoogleOnceInit(&repeated_string_type_traits_once_init_, &InitializeDefaultRepeatedFields); return default_repeated_field_; } @@ -1034,8 +1032,7 @@ class RepeatedMessageTypeTraits { static const RepeatedFieldType* GetDefaultRepeatedField(); }; -LIBPROTOBUF_EXPORT extern ProtobufOnceType -repeated_message_generic_type_traits_once_init_; +extern ProtobufOnceType repeated_message_generic_type_traits_once_init_; // This class exists only to hold a generic default empty repeated field for all // message-type repeated field extensions. @@ -1052,7 +1049,7 @@ class LIBPROTOBUF_EXPORT RepeatedMessageGenericTypeTraits { template inline const typename RepeatedMessageTypeTraits::RepeatedFieldType* RepeatedMessageTypeTraits::GetDefaultRepeatedField() { - GoogleOnceInit( + ::google::protobuf::GoogleOnceInit( &repeated_message_generic_type_traits_once_init_, &RepeatedMessageGenericTypeTraits::InitializeDefaultRepeatedFields); return reinterpret_cast( diff --git a/src/google/protobuf/extension_set_unittest.cc b/src/google/protobuf/extension_set_unittest.cc index 1569120d15..f40fcbc26d 100644 --- a/src/google/protobuf/extension_set_unittest.cc +++ b/src/google/protobuf/extension_set_unittest.cc @@ -360,9 +360,8 @@ TEST(ExtensionSetTest, ArenaSetAllocatedMessageAndRelease) { unittest::ForeignMessage* foreign_message = new unittest::ForeignMessage(); message->SetAllocatedExtension(unittest::optional_foreign_message_extension, foreign_message); - // foreign_message is copied underneath, as foreign_message is on heap - // and extension_set is on an arena. - EXPECT_NE(foreign_message, + // foreign_message is now owned by the arena. + EXPECT_EQ(foreign_message, message->MutableExtension( unittest::optional_foreign_message_extension)); diff --git a/src/google/protobuf/generated_message_reflection_unittest.cc b/src/google/protobuf/generated_message_reflection_unittest.cc index df04384481..85ebdef1fc 100644 --- a/src/google/protobuf/generated_message_reflection_unittest.cc +++ b/src/google/protobuf/generated_message_reflection_unittest.cc @@ -43,6 +43,11 @@ // rather than generated accessors. #include +#include +#ifndef _SHARED_PTR_H +#include +#endif + #include #include #include @@ -354,7 +359,7 @@ TEST(GeneratedMessageReflectionTest, ReleaseLast) { ASSERT_EQ(2, message.repeated_foreign_message_size()); const protobuf_unittest::ForeignMessage* expected = message.mutable_repeated_foreign_message(1); - scoped_ptr released(message.GetReflection()->ReleaseLast( + google::protobuf::scoped_ptr released(message.GetReflection()->ReleaseLast( &message, descriptor->FindFieldByName("repeated_foreign_message"))); EXPECT_EQ(expected, released.get()); } @@ -377,9 +382,9 @@ TEST(GeneratedMessageReflectionTest, ReleaseLastExtensions) { unittest::repeated_foreign_message_extension)); const protobuf_unittest::ForeignMessage* expected = message.MutableExtension( unittest::repeated_foreign_message_extension, 1); - scoped_ptr released(message.GetReflection()->ReleaseLast( + google::protobuf::scoped_ptr released(message.GetReflection()->ReleaseLast( &message, descriptor->file()->FindExtensionByName( - "repeated_foreign_message_extension"))); + "repeated_foreign_message_extension"))); EXPECT_EQ(expected, released.get()); } diff --git a/src/google/protobuf/generated_message_util.cc b/src/google/protobuf/generated_message_util.cc index afaca2ee3f..7b813f8af5 100644 --- a/src/google/protobuf/generated_message_util.cc +++ b/src/google/protobuf/generated_message_util.cc @@ -63,7 +63,6 @@ void InitEmptyString() { int StringSpaceUsedExcludingSelf(const string& str) { const void* start = &str; const void* end = &str + 1; - if (start <= str.data() && str.data() < end) { // The string's data is stored inside the string object itself. return 0; @@ -73,6 +72,7 @@ int StringSpaceUsedExcludingSelf(const string& str) { } + } // namespace internal } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/io/coded_stream_unittest.cc b/src/google/protobuf/io/coded_stream_unittest.cc index 630c908690..d1782e39a9 100644 --- a/src/google/protobuf/io/coded_stream_unittest.cc +++ b/src/google/protobuf/io/coded_stream_unittest.cc @@ -34,6 +34,10 @@ // // This file contains tests and benchmarks. +#include +#ifndef _SHARED_PTR_H +#include +#endif #include #include @@ -679,7 +683,7 @@ TEST_F(CodedStreamTest, ReadStringImpossiblyLargeFromStringOnStack) { } TEST_F(CodedStreamTest, ReadStringImpossiblyLargeFromStringOnHeap) { - scoped_array buffer(new uint8[8]); + google::protobuf::scoped_array buffer(new uint8[8]); CodedInputStream coded_input(buffer.get(), 8); string str; EXPECT_FALSE(coded_input.ReadString(&str, 1 << 30)); diff --git a/src/google/protobuf/io/zero_copy_stream_impl_lite.h b/src/google/protobuf/io/zero_copy_stream_impl_lite.h index 9cdf037854..a598ef2e3d 100644 --- a/src/google/protobuf/io/zero_copy_stream_impl_lite.h +++ b/src/google/protobuf/io/zero_copy_stream_impl_lite.h @@ -44,6 +44,10 @@ #ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__ #define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__ +#include +#ifndef _SHARED_PTR_H +#include +#endif #include #include #include @@ -235,7 +239,7 @@ class LIBPROTOBUF_EXPORT CopyingInputStreamAdaptor : public ZeroCopyInputStream // Data is read into this buffer. It may be NULL if no buffer is currently // in use. Otherwise, it points to an array of size buffer_size_. - scoped_array buffer_; + google::protobuf::scoped_array buffer_; const int buffer_size_; // Number of valid bytes currently in the buffer (i.e. the size last @@ -324,7 +328,7 @@ class LIBPROTOBUF_EXPORT CopyingOutputStreamAdaptor : public ZeroCopyOutputStrea // Data is written from this buffer. It may be NULL if no buffer is // currently in use. Otherwise, it points to an array of size buffer_size_. - scoped_array buffer_; + google::protobuf::scoped_array buffer_; const int buffer_size_; // Number of valid bytes currently in the buffer (i.e. the size last diff --git a/src/google/protobuf/io/zero_copy_stream_unittest.cc b/src/google/protobuf/io/zero_copy_stream_unittest.cc index 3850e76c40..8c7358c191 100644 --- a/src/google/protobuf/io/zero_copy_stream_unittest.cc +++ b/src/google/protobuf/io/zero_copy_stream_unittest.cc @@ -57,6 +57,10 @@ #include #include #include +#include +#ifndef _SHARED_PTR_H +#include +#endif #include #include @@ -197,7 +201,7 @@ void IoTest::WriteString(ZeroCopyOutputStream* output, const string& str) { } void IoTest::ReadString(ZeroCopyInputStream* input, const string& str) { - scoped_array buffer(new char[str.size() + 1]); + google::protobuf::scoped_array buffer(new char[str.size() + 1]); buffer[str.size()] = '\0'; EXPECT_EQ(ReadFromInput(input, buffer.get(), str.size()), str.size()); EXPECT_STREQ(str.c_str(), buffer.get()); diff --git a/src/google/protobuf/map_field_inl.h b/src/google/protobuf/map_field_inl.h index 16c4a08ffc..f116697c43 100644 --- a/src/google/protobuf/map_field_inl.h +++ b/src/google/protobuf/map_field_inl.h @@ -155,7 +155,8 @@ void TypeDefinedMapFieldBase::CopyIterator( this_iter->key_.SetType(that_iter.key_.type()); // MapValueRef::type() fails when containing data is null. However, if // this_iter points to MapEnd, data can be null. - this_iter->value_.SetType((FieldDescriptor::CppType)that_iter.value_.type_); + this_iter->value_.SetType( + static_cast(that_iter.value_.type_)); SetMapIteratorValue(this_iter); } diff --git a/src/google/protobuf/map_proto2_unittest.proto b/src/google/protobuf/map_proto2_unittest.proto index 6f9d6165c6..916cc54637 100644 --- a/src/google/protobuf/map_proto2_unittest.proto +++ b/src/google/protobuf/map_proto2_unittest.proto @@ -31,6 +31,8 @@ syntax = "proto2"; +import "google/protobuf/unittest_import.proto"; + // We don't put this in a package within proto2 because we need to make sure // that the generated code doesn't depend on being in the proto2 namespace. // In map_test_util.h we do "using namespace unittest = protobuf_unittest". @@ -58,3 +60,7 @@ message TestEnumMapPlusExtra { map known_map_field = 101; map unknown_map_field = 102; } + +message TestImportEnumMap { + map import_enum_amp = 1; +} diff --git a/src/google/protobuf/map_test.cc b/src/google/protobuf/map_test.cc index 16a24c25ab..451b02e84a 100644 --- a/src/google/protobuf/map_test.cc +++ b/src/google/protobuf/map_test.cc @@ -2154,7 +2154,7 @@ TEST_F(MapFieldInDynamicMessageTest, MapIndependentOffsets) { // Check that all fields have independent offsets by setting each // one to a unique value then checking that they all still have those // unique values (i.e. they don't stomp each other). - scoped_ptr message(map_prototype_->New()); + google::protobuf::scoped_ptr message(map_prototype_->New()); MapReflectionTester reflection_tester(map_descriptor_); reflection_tester.SetMapFieldsViaReflection(message.get()); @@ -2163,7 +2163,7 @@ TEST_F(MapFieldInDynamicMessageTest, MapIndependentOffsets) { TEST_F(MapFieldInDynamicMessageTest, DynamicMapReflection) { // Check that map fields work properly. - scoped_ptr message(map_prototype_->New()); + google::protobuf::scoped_ptr message(map_prototype_->New()); // Check set functions. MapReflectionTester reflection_tester(map_descriptor_); @@ -2177,7 +2177,7 @@ TEST_F(MapFieldInDynamicMessageTest, MapSpaceUsed) { // Since we share the implementation with generated messages, we don't need // to test very much here. Just make sure it appears to be working. - scoped_ptr message(map_prototype_->New()); + google::protobuf::scoped_ptr message(map_prototype_->New()); MapReflectionTester reflection_tester(map_descriptor_); int initial_space_used = message->SpaceUsed(); diff --git a/src/google/protobuf/message.cc b/src/google/protobuf/message.cc index 2f6416d0f3..d9c200c65b 100644 --- a/src/google/protobuf/message.cc +++ b/src/google/protobuf/message.cc @@ -55,7 +55,6 @@ #include #include #include -#include namespace google { namespace protobuf { diff --git a/src/google/protobuf/message.h b/src/google/protobuf/message.h index 348e7c7fc3..7c27afd999 100644 --- a/src/google/protobuf/message.h +++ b/src/google/protobuf/message.h @@ -971,7 +971,7 @@ class LIBPROTOBUF_EXPORT Reflection { return false; } - // Returns a MaIterator referring to the first element in the map field. + // Returns a MapIterator referring to the first element in the map field. // If the map field is empty, this function returns the same as // reflection::MapEnd. Mutation to the field may invalidate the iterator. virtual MapIterator MapBegin( diff --git a/src/google/protobuf/repeated_field.h b/src/google/protobuf/repeated_field.h index b10e7a9589..5530fefe1a 100644 --- a/src/google/protobuf/repeated_field.h +++ b/src/google/protobuf/repeated_field.h @@ -649,7 +649,7 @@ inline const Message& GenericTypeHandler::default_instance() { // StringTypeHandler is exported. So, we factor out StringTypeHandlerBase, // export that, then make StringTypeHandler be a subclass which is NOT // exported. -// TODO(kenton): Now that StringSpaceUsedExcludingSelf() is in the lite +// TODO(kenton): Now that StringSpaceUsedExcludingSelf() is in the lite // library, this can be cleaned up. class LIBPROTOBUF_EXPORT StringTypeHandlerBase { public: diff --git a/src/google/protobuf/stubs/stringpiece.h b/src/google/protobuf/stubs/stringpiece.h index 353a60d37c..ec3ffd5b06 100644 --- a/src/google/protobuf/stubs/stringpiece.h +++ b/src/google/protobuf/stubs/stringpiece.h @@ -149,6 +149,7 @@ #include #include +#include namespace google { namespace protobuf { @@ -437,4 +438,16 @@ extern std::ostream& operator<<(std::ostream& o, StringPiece piece); } // namespace protobuf } // namespace google +GOOGLE_PROTOBUF_HASH_NAMESPACE_DECLARATION_START +template<> struct hash { + size_t operator()(const StringPiece& s) const { + size_t result = 0; + for (const char *str = s.data(), *end = str + s.size(); str < end; str++) { + result = 5 * result + *str; + } + return result; + } +}; +GOOGLE_PROTOBUF_HASH_NAMESPACE_DECLARATION_END + #endif // STRINGS_STRINGPIECE_H_ diff --git a/src/google/protobuf/stubs/stringpiece_unittest.cc b/src/google/protobuf/stubs/stringpiece_unittest.cc index 9b5dae1352..1cb7d12b0b 100644 --- a/src/google/protobuf/stubs/stringpiece_unittest.cc +++ b/src/google/protobuf/stubs/stringpiece_unittest.cc @@ -30,6 +30,7 @@ #include #include +#include #include #include #include @@ -745,6 +746,23 @@ TEST(StringPiece, Comparisons2) { EXPECT_TRUE(abc.ends_with("nopqrstuvwxyz")); } +TEST(StringPiece, HashFunction) { + hash_set set; + + set.insert(StringPiece("hello")); + EXPECT_EQ(1, set.size()); + + // Insert a StringPiece of the same value again and should NOT increment + // size of the set. + set.insert(StringPiece("hello")); + EXPECT_EQ(1, set.size()); + + // Insert a StringPiece with different value and check that size of the set + // has been increment by one. + set.insert(StringPiece("world")); + EXPECT_EQ(2, set.size()); +} + TEST(ComparisonOpsTest, StringCompareNotAmbiguous) { EXPECT_EQ("hello", string("hello")); EXPECT_LT("hello", string("world")); diff --git a/src/google/protobuf/text_format.cc b/src/google/protobuf/text_format.cc index 4d8c1f91dd..38b9069bcc 100644 --- a/src/google/protobuf/text_format.cc +++ b/src/google/protobuf/text_format.cc @@ -377,8 +377,8 @@ class TextFormat::Parser::ParserImpl { if (internal::GetAnyFieldDescriptors(*message, &any_type_url_field, &any_value_field) && TryConsume("[")) { - string full_type_name; - DO(ConsumeAnyTypeUrl(&full_type_name)); + string full_type_name, prefix; + DO(ConsumeAnyTypeUrl(&full_type_name, &prefix)); DO(Consume("]")); string serialized_value; DO(ConsumeAnyValue(full_type_name, @@ -386,7 +386,7 @@ class TextFormat::Parser::ParserImpl { &serialized_value)); reflection->SetString( message, any_type_url_field, - string(internal::kTypeGoogleApisComPrefix) + full_type_name); + string(prefix + full_type_name)); reflection->SetString(message, any_value_field, serialized_value); return true; // Fall through. @@ -981,7 +981,8 @@ class TextFormat::Parser::ParserImpl { } // Consumes Any::type_url value, of form "type.googleapis.com/full.type.Name" - bool ConsumeAnyTypeUrl(string* full_type_name) { + // or "type.googleprod.com/full.type.Name" + bool ConsumeAnyTypeUrl(string* full_type_name, string* prefix) { // TODO(saito) Extend Consume() to consume multiple tokens at once, so that // this code can be written as just DO(Consume(kGoogleApisTypePrefix)). string url1, url2, url3; @@ -993,10 +994,12 @@ class TextFormat::Parser::ParserImpl { DO(Consume("/")); DO(ConsumeFullTypeName(full_type_name)); - const string prefix = url1 + "." + url2 + "." + url3 + "/"; - if (prefix != internal::kTypeGoogleApisComPrefix) { + *prefix = url1 + "." + url2 + "." + url3 + "/"; + if (*prefix != internal::kTypeGoogleApisComPrefix && + *prefix != internal::kTypeGoogleProdComPrefix) { ReportError("TextFormat::Parser for Any supports only " - "type.googleapi.com, but found \"" + prefix + "\""); + "type.googleapis.com and type.googleprod.com, " + "but found \"" + *prefix + "\""); return false; } return true; diff --git a/src/google/protobuf/text_format_unittest.cc b/src/google/protobuf/text_format_unittest.cc index 1aafd8e6e7..8d61be1924 100644 --- a/src/google/protobuf/text_format_unittest.cc +++ b/src/google/protobuf/text_format_unittest.cc @@ -37,6 +37,10 @@ #include #include #include +#include +#ifndef _SHARED_PTR_H +#include +#endif #include #include @@ -931,7 +935,7 @@ class TextFormatParserTest : public testing::Test { protected: void ExpectFailure(const string& input, const string& message, int line, int col) { - scoped_ptr proto(new unittest::TestAllTypes); + google::protobuf::scoped_ptr proto(new unittest::TestAllTypes); ExpectFailure(input, message, line, col, proto.get()); } @@ -992,7 +996,7 @@ class TextFormatParserTest : public testing::Test { }; TEST_F(TextFormatParserTest, ParseInfoTreeBuilding) { - scoped_ptr message(new unittest::TestAllTypes); + google::protobuf::scoped_ptr message(new unittest::TestAllTypes); const Descriptor* d = message->GetDescriptor(); string stringData = @@ -1057,7 +1061,7 @@ TEST_F(TextFormatParserTest, ParseInfoTreeBuilding) { } TEST_F(TextFormatParserTest, ParseFieldValueFromString) { - scoped_ptr message(new unittest::TestAllTypes); + google::protobuf::scoped_ptr message(new unittest::TestAllTypes); const Descriptor* d = message->GetDescriptor(); #define EXPECT_FIELD(name, value, valuestring) \ diff --git a/src/google/protobuf/unittest_import.proto b/src/google/protobuf/unittest_import.proto index 7e16522033..8d03e3888b 100644 --- a/src/google/protobuf/unittest_import.proto +++ b/src/google/protobuf/unittest_import.proto @@ -64,3 +64,10 @@ enum ImportEnum { IMPORT_BAZ = 9; } + +// To use an enum in a map, it must has the first value as 0. +enum ImportEnumForMap { + UNKNOWN = 0; + FOO = 1; + BAR = 2; +} diff --git a/src/google/protobuf/util/default_value_objectwriter.h b/src/google/protobuf/util/default_value_objectwriter.h new file mode 100644 index 0000000000..11b9f801e3 --- /dev/null +++ b/src/google/protobuf/util/default_value_objectwriter.h @@ -0,0 +1,218 @@ +#ifndef NET_PROTO2_UTIL_CONVERTER_INTERNAL_DEFAULT_VALUE_OBJECTWRITER_H_ +#define NET_PROTO2_UTIL_CONVERTER_INTERNAL_DEFAULT_VALUE_OBJECTWRITER_H_ + +#include +#include +#include + +#include "base/macros.h" +#include "net/proto2/util/converter/internal/type_info.h" +#include "net/proto2/util/converter/public/datapiece.h" +#include "net/proto2/util/converter/public/object_writer.h" +#include "net/proto2/util/converter/public/utility.h" +#include "net/proto2/util/public/type_resolver.h" +#include "strings/stringpiece.h" + +namespace proto2 { +namespace util { +namespace converter { + +// An ObjectWriter that renders non-repeated primitive fields of proto messages +// with their default values. DefaultValueObjectWriter holds objects, lists and +// fields it receives in a tree structure and writes them out to another +// ObjectWriter when EndObject() is called on the root object. It also writes +// out all non-repeated primitive fields that haven't been explicitly rendered +// with their default values (0 for numbers, "" for strings, etc). +class DefaultValueObjectWriter : public ObjectWriter { + public: +#ifndef PROTO2_OPENSOURCE + DefaultValueObjectWriter(const TypeInfo& typeinfo, + const google::protobuf::Type& type, + ObjectWriter* ow); +#endif // !PROTO2_OPENSOURCE + DefaultValueObjectWriter(TypeResolver* type_resolver, + const google::protobuf::Type& type, + ObjectWriter* ow); + + virtual ~DefaultValueObjectWriter(); + + // ObjectWriter methods. + virtual DefaultValueObjectWriter* StartObject(StringPiece name); + + virtual DefaultValueObjectWriter* EndObject(); + + virtual DefaultValueObjectWriter* StartList(StringPiece name); + + virtual DefaultValueObjectWriter* EndList(); + + virtual DefaultValueObjectWriter* RenderBool(StringPiece name, bool value); + + virtual DefaultValueObjectWriter* RenderInt32(StringPiece name, int32 value); + + virtual DefaultValueObjectWriter* RenderUint32(StringPiece name, + uint32 value); + + virtual DefaultValueObjectWriter* RenderInt64(StringPiece name, int64 value); + + virtual DefaultValueObjectWriter* RenderUint64(StringPiece name, + uint64 value); + + virtual DefaultValueObjectWriter* RenderDouble(StringPiece name, + double value); + + virtual DefaultValueObjectWriter* RenderFloat(StringPiece name, float value); + + virtual DefaultValueObjectWriter* RenderString(StringPiece name, + StringPiece value); +#ifdef PROTO2_OPENSOURCE + virtual DefaultValueObjectWriter* RenderBytes(StringPiece name, + StringPiece value); +#else // PROTO2_OPENSOURCE + virtual DefaultValueObjectWriter* RenderCord(StringPiece name, + const Cord& value); +#endif // !PROTO2_OPENSOURCE + + virtual DefaultValueObjectWriter* RenderNull(StringPiece name); + + virtual DefaultValueObjectWriter* DisableCaseNormalizationForNextKey(); + + private: + enum NodeKind { + PRIMITIVE = 0, + OBJECT = 1, + LIST = 2, + MAP = 3, + }; + + // "Node" represents a node in the tree that holds the input of + // DefaultValueObjectWriter. + class Node { + public: + Node(const string& name, const google::protobuf::Type* type, NodeKind kind, + const DataPiece& data, bool is_placeholder); + virtual ~Node() { + for (int i = 0; i < children_.size(); ++i) { + delete children_[i]; + } + } + + // Adds a child to this node. Takes ownership of this child. + void AddChild(Node* child) { children_.push_back(child); } + + // Finds the child given its name. + Node* FindChild(StringPiece name); + + // Populates children of this Node based on its type. If there are already + // children created, they will be merged to the result. Caller should pass + // in TypeInfo for looking up types of the children. + void PopulateChildren(const TypeInfo* typeinfo); + + // If this node is a leaf (has data), writes the current node to the + // ObjectWriter; if not, then recursively writes the children to the + // ObjectWriter. + void WriteTo(ObjectWriter* ow); + + // Accessors + const string& name() const { return name_; } + + const google::protobuf::Type* type() { return type_; } + + void set_type(const google::protobuf::Type* type) { type_ = type; } + + NodeKind kind() { return kind_; } + + int number_of_children() { return children_.size(); } + + void set_data(const DataPiece& data) { data_ = data; } + + void set_disable_normalize(bool disable_normalize) { + disable_normalize_ = disable_normalize; + } + + bool is_any() { return is_any_; } + + void set_is_any(bool is_any) { is_any_ = is_any; } + + void set_is_placeholder(bool is_placeholder) { + is_placeholder_ = is_placeholder; + } + + private: + // Returns the Value Type of a map given the Type of the map entry and a + // TypeInfo instance. + const google::protobuf::Type* GetMapValueType( + const google::protobuf::Type& entry_type, const TypeInfo* typeinfo); + + // Calls WriteTo() on every child in children_. + void WriteChildren(ObjectWriter* ow); + + // The name of this node. + string name_; + // google::protobuf::Type of this node. Owned by TypeInfo. + const google::protobuf::Type* type_; + // The kind of this node. + NodeKind kind_; + // Whether to disable case normalization of the name. + bool disable_normalize_; + // Whether this is a node for "Any". + bool is_any_; + // The data of this node when it is a leaf node. + DataPiece data_; + // Children of this node. + std::vector children_; + // Whether this node is a placeholder for an object or list automatically + // generated when creating the parent node. Should be set to false after + // the parent node's StartObject()/StartList() method is called with this + // node's name. + bool is_placeholder_; + }; + + // Populates children of "node" if it is an "any" Node and its real type has + // been given. + void MaybePopulateChildrenOfAny(Node* node); + + // Writes the root_ node to ow_ and resets the root_ and current_ pointer to + // nullptr. + void WriteRoot(); + + // Creates a DataPiece containing the default value of the type of the field. + static DataPiece CreateDefaultDataPieceForField( + const google::protobuf::Field& field); + + // Returns disable_normalize_ and reset it to false. + bool GetAndResetDisableNormalize() { + return disable_normalize_ ? (disable_normalize_ = false, true) : false; + } + + // Adds or replaces the data_ of a primitive child node. + void RenderDataPiece(StringPiece name, const DataPiece& data); + + // Type information for all the types used in the descriptor. Used to find + // google::protobuf::Type of nested messages/enums. + const TypeInfo* typeinfo_; + // Whether the TypeInfo object is owned by this class. + bool own_typeinfo_; + // google::protobuf::Type of the root message type. + const google::protobuf::Type& type_; + // Holds copies of strings passed to RenderString. + vector string_values_; + + // Whether to disable case normalization of the next node. + bool disable_normalize_; + // The current Node. Owned by its parents. + Node* current_; + // The root Node. + std::unique_ptr root_; + // The stack to hold the path of Nodes from current_ to root_; + std::stack stack_; + + ObjectWriter* ow_; + + DISALLOW_COPY_AND_ASSIGN(DefaultValueObjectWriter); +}; + +} // namespace converter +} // namespace util +} // namespace proto2 + +#endif // NET_PROTO2_UTIL_CONVERTER_INTERNAL_DEFAULT_VALUE_OBJECTWRITER_H_ diff --git a/src/google/protobuf/util/field_comparator_test.cc b/src/google/protobuf/util/field_comparator_test.cc index 845839acdf..748c7d11c8 100644 --- a/src/google/protobuf/util/field_comparator_test.cc +++ b/src/google/protobuf/util/field_comparator_test.cc @@ -34,8 +34,8 @@ #include #include -#include #include +#include namespace google { namespace protobuf { diff --git a/src/google/protobuf/util/field_mask_util.cc b/src/google/protobuf/util/field_mask_util.cc index 82034bd412..29ca9c1e99 100644 --- a/src/google/protobuf/util/field_mask_util.cc +++ b/src/google/protobuf/util/field_mask_util.cc @@ -43,7 +43,7 @@ string FieldMaskUtil::ToString(const FieldMask& mask) { return Join(mask.paths(), ","); } -void FieldMaskUtil::FromString(const string& str, FieldMask* out) { +void FieldMaskUtil::FromString(StringPiece str, FieldMask* out) { out->Clear(); vector paths = Split(str, ","); for (int i = 0; i < paths.size(); ++i) { @@ -53,7 +53,7 @@ void FieldMaskUtil::FromString(const string& str, FieldMask* out) { } bool FieldMaskUtil::InternalIsValidPath(const Descriptor* descriptor, - const string& path) { + StringPiece path) { vector parts = Split(path, "."); for (int i = 0; i < parts.size(); ++i) { const string& field_name = parts[i]; @@ -386,15 +386,15 @@ void FieldMaskUtil::Intersect(const FieldMask& mask1, const FieldMask& mask2, intersection.MergeToFieldMask(out); } -bool FieldMaskUtil::IsPathInFieldMask(const string& path, - const FieldMask& mask) { +bool FieldMaskUtil::IsPathInFieldMask(StringPiece path, const FieldMask& mask) { for (int i = 0; i < mask.paths_size(); ++i) { const string& mask_path = mask.paths(i); if (path == mask_path) { return true; } else if (mask_path.length() < path.length()) { // Also check whether mask.paths(i) is a prefix of path. - if (path.compare(0, mask_path.length() + 1, mask_path + ".") == 0) { + if (path.substr(0, mask_path.length() + 1).compare(mask_path + ".") == + 0) { return true; } } diff --git a/src/google/protobuf/util/field_mask_util.h b/src/google/protobuf/util/field_mask_util.h index c99c34f87f..92f69893a0 100644 --- a/src/google/protobuf/util/field_mask_util.h +++ b/src/google/protobuf/util/field_mask_util.h @@ -35,6 +35,7 @@ #include #include +#include namespace google { namespace protobuf { @@ -47,11 +48,11 @@ class LIBPROTOBUF_EXPORT FieldMaskUtil { // Converts FieldMask to/from string, formatted according to proto3 JSON // spec for FieldMask (e.g., "foo,bar,baz.quz"). static string ToString(const FieldMask& mask); - static void FromString(const string& str, FieldMask* out); + static void FromString(StringPiece str, FieldMask* out); // Checks whether the given path is valid for type T. template - static bool IsValidPath(const string& path) { + static bool IsValidPath(StringPiece path) { return InternalIsValidPath(T::descriptor(), path); } @@ -67,7 +68,7 @@ class LIBPROTOBUF_EXPORT FieldMaskUtil { // Adds a path to FieldMask after checking whether the given path is valid. // This method check-fails if the path is not a valid path for type T. template - static void AddPathToFieldMask(const string& path, FieldMask* mask) { + static void AddPathToFieldMask(StringPiece path, FieldMask* mask) { GOOGLE_CHECK(IsValidPath(path)); mask->add_paths(path); } @@ -96,7 +97,7 @@ class LIBPROTOBUF_EXPORT FieldMaskUtil { // Returns true if path is covered by the given FieldMask. Note that path // "foo.bar" covers all paths like "foo.bar.baz", "foo.bar.quz.x", etc. - static bool IsPathInFieldMask(const string& path, const FieldMask& mask); + static bool IsPathInFieldMask(StringPiece path, const FieldMask& mask); class MergeOptions; // Merges fields specified in a FieldMask into another message. @@ -105,7 +106,7 @@ class LIBPROTOBUF_EXPORT FieldMaskUtil { private: static bool InternalIsValidPath(const Descriptor* descriptor, - const string& path); + StringPiece path); static void InternalGetFieldMaskForAllFields(const Descriptor* descriptor, FieldMask* out); diff --git a/src/google/protobuf/util/internal/default_value_objectwriter.h b/src/google/protobuf/util/internal/default_value_objectwriter.h index d454760170..bcb526e8f9 100644 --- a/src/google/protobuf/util/internal/default_value_objectwriter.h +++ b/src/google/protobuf/util/internal/default_value_objectwriter.h @@ -110,7 +110,7 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { // "Node" represents a node in the tree that holds the input of // DefaultValueObjectWriter. - class Node { + class LIBPROTOBUF_EXPORT Node { public: Node(const string& name, const google::protobuf::Type* type, NodeKind kind, const DataPiece& data, bool is_placeholder); diff --git a/src/google/protobuf/util/internal/json_objectwriter.cc b/src/google/protobuf/util/internal/json_objectwriter.cc index f81e3306ac..94d2ab7a48 100644 --- a/src/google/protobuf/util/internal/json_objectwriter.cc +++ b/src/google/protobuf/util/internal/json_objectwriter.cc @@ -37,8 +37,8 @@ #include #include #include -#include #include +#include namespace google { namespace protobuf { @@ -116,7 +116,7 @@ JsonObjectWriter* JsonObjectWriter::RenderUint64(StringPiece name, JsonObjectWriter* JsonObjectWriter::RenderDouble(StringPiece name, double value) { - if (google::protobuf::MathLimits::IsFinite(value)) { + if (MathLimits::IsFinite(value)) { return RenderSimple(name, SimpleDtoa(value)); } @@ -126,7 +126,7 @@ JsonObjectWriter* JsonObjectWriter::RenderDouble(StringPiece name, JsonObjectWriter* JsonObjectWriter::RenderFloat(StringPiece name, float value) { - if (google::protobuf::MathLimits::IsFinite(value)) { + if (MathLimits::IsFinite(value)) { return RenderSimple(name, SimpleFtoa(value)); } diff --git a/src/google/protobuf/util/internal/protostream_objectsource.cc b/src/google/protobuf/util/internal/protostream_objectsource.cc index 996e1f8cd0..aebf19a1c5 100644 --- a/src/google/protobuf/util/internal/protostream_objectsource.cc +++ b/src/google/protobuf/util/internal/protostream_objectsource.cc @@ -47,7 +47,6 @@ #include #include #include -#include #include @@ -140,6 +139,7 @@ Status ProtoStreamObjectSource::WriteMessage(const google::protobuf::Type& type, const uint32 end_tag, bool include_start_and_end, ObjectWriter* ow) const { + const TypeRenderer* type_renderer = FindTypeRenderer(type.name()); if (type_renderer != NULL) { return (*type_renderer)(this, type, name, ow); @@ -332,10 +332,9 @@ Status ProtoStreamObjectSource::RenderDuration( if (seconds < 0) { if (nanos > 0) { return Status(util::error::INTERNAL, - StrCat( - "Duration nanos is non-negative, but seconds is " - "negative for field: ", - field_name)); + StrCat("Duration nanos is non-negative, but seconds is " + "negative for field: ", + field_name)); } sign = "-"; seconds = -seconds; @@ -648,6 +647,7 @@ Status ProtoStreamObjectSource::RenderFieldMask( return Status::OK; } + hash_map* ProtoStreamObjectSource::renderers_ = NULL; GOOGLE_PROTOBUF_DECLARE_ONCE(source_renderers_init_); @@ -670,13 +670,16 @@ void ProtoStreamObjectSource::InitRendererMap() { &ProtoStreamObjectSource::RenderInt32; (*renderers_)["google.protobuf.UInt32Value"] = &ProtoStreamObjectSource::RenderUInt32; - (*renderers_)["google.protobuf.BoolValue"] = &ProtoStreamObjectSource::RenderBool; + (*renderers_)["google.protobuf.BoolValue"] = + &ProtoStreamObjectSource::RenderBool; (*renderers_)["google.protobuf.StringValue"] = &ProtoStreamObjectSource::RenderString; (*renderers_)["google.protobuf.BytesValue"] = &ProtoStreamObjectSource::RenderBytes; - (*renderers_)["google.protobuf.Any"] = &ProtoStreamObjectSource::RenderAny; - (*renderers_)["google.protobuf.Struct"] = &ProtoStreamObjectSource::RenderStruct; + (*renderers_)["google.protobuf.Any"] = + &ProtoStreamObjectSource::RenderAny; + (*renderers_)["google.protobuf.Struct"] = + &ProtoStreamObjectSource::RenderStruct; (*renderers_)["google.protobuf.Value"] = &ProtoStreamObjectSource::RenderStructValue; (*renderers_)["google.protobuf.ListValue"] = @@ -835,6 +838,7 @@ Status ProtoStreamObjectSource::RenderField( StrCat("Invalid configuration. Could not find the type: ", field->type_url())); } + RETURN_IF_ERROR(WriteMessage(*type, field_name, 0, true, ow)); if (!stream_->ConsumedEntireMessage()) { return Status(util::error::INVALID_ARGUMENT, @@ -988,7 +992,7 @@ std::pair ProtoStreamObjectSource::ReadSecondsAndNanos( uint32 nanos = 0; uint32 tag = 0; int64 signed_seconds = 0; - int64 signed_nanos = 0; + int32 signed_nanos = 0; for (tag = stream_->ReadTag(); tag != 0; tag = stream_->ReadTag()) { const google::protobuf::Field* field = FindAndVerifyField(type, tag); diff --git a/src/google/protobuf/util/internal/protostream_objectsource.h b/src/google/protobuf/util/internal/protostream_objectsource.h index f52383a171..3cd37aa180 100644 --- a/src/google/protobuf/util/internal/protostream_objectsource.h +++ b/src/google/protobuf/util/internal/protostream_objectsource.h @@ -188,8 +188,7 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource { // Helper to render google.protobuf.Struct's ListValue fields to ObjectWriter. static util::Status RenderStructListValue( - const ProtoStreamObjectSource* os, - const google::protobuf::Type& type, + const ProtoStreamObjectSource* os, const google::protobuf::Type& type, StringPiece name, ObjectWriter* ow); // Render the "Any" type. @@ -211,6 +210,7 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource { util::Status RenderField(const google::protobuf::Field* field, StringPiece field_name, ObjectWriter* ow) const; + // Reads field value according to Field spec in 'field' and returns the read // value as string. This only works for primitive datatypes (no message // types). diff --git a/src/google/protobuf/util/internal/protostream_objectwriter.cc b/src/google/protobuf/util/internal/protostream_objectwriter.cc index a935ac390a..bd24599927 100644 --- a/src/google/protobuf/util/internal/protostream_objectwriter.cc +++ b/src/google/protobuf/util/internal/protostream_objectwriter.cc @@ -33,13 +33,13 @@ #include #include +#include #include #include #include #include #include #include -#include #include #include #include @@ -397,8 +397,8 @@ void ProtoStreamObjectWriter::AnyWriter::RenderDataPiece( const TypeRenderer* type_renderer = FindTypeRenderer(GetFullTypeWithUrl(ow_->master_type_.name())); if (type_renderer) { - // TODO(rikka): Don't just ignore the util::Status object! - (*type_renderer)(ow_.get(), value); + Status status = (*type_renderer)(ow_.get(), value); + if (!status.ok()) ow_->InvalidValue("Any", status.error_message()); } else { ow_->RenderDataPiece(name, value); } @@ -600,6 +600,11 @@ void ProtoStreamObjectWriter::ProtoElement::TakeOneofIndex(int32 index) { InsertIfNotPresent(&oneof_indices_, index); } +bool ProtoStreamObjectWriter::ProtoElement::InsertMapKeyIfNotPresent( + StringPiece map_key) { + return InsertIfNotPresent(&map_keys_, map_key); +} + inline void ProtoStreamObjectWriter::InvalidName(StringPiece unknown_name, StringPiece message) { listener_->InvalidName(location(), ToSnakeCase(unknown_name), message); @@ -643,6 +648,11 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartObject( return this; } else if (element_ != NULL && (element_->IsMap() || element_->IsStructMap())) { + if (!ValidMapKey(name)) { + ++invalid_depth_; + return this; + } + field = StartMapEntry(name); if (element_->IsStructMapEntry()) { // If the top element is a map entry, this means we are starting another @@ -698,7 +708,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartObject( element_.reset( new ProtoElement(element_.release(), field, *type, ProtoElement::MAP)); } else { - WriteTag(*field); + WriteTag(*field); element_.reset(new ProtoElement(element_.release(), field, *type, ProtoElement::MESSAGE)); } @@ -863,6 +873,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::EndObject() { // struct. SkipElements(); + // If ending the root element, // then serialize the full message with calculated sizes. if (element_ == NULL) { @@ -914,6 +925,11 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartList(StringPiece name) { // Check if we need to start a map. This can heppen when there is either a map // or a struct type within a list. if (element_->IsMap() || element_->IsStructMap()) { + if (!ValidMapKey(name)) { + ++invalid_depth_; + return this; + } + field = StartMapEntry(name); if (field == NULL) return this; @@ -969,7 +985,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartList(StringPiece name) { ProtoElement::MESSAGE)); InvalidValue("Map", "Cannot bind a list to map."); ++invalid_depth_; - element_->pop(); + element_.reset(element_->pop()); return this; } @@ -1188,6 +1204,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::RenderDataPiece( type_url = GetFullTypeWithUrl(master_type_.name()); } else { if (element_->IsMap() || element_->IsStructMap()) { + if (!ValidMapKey(name)) return this; is_map_entry = true; field = StartMapEntry(name); } else { @@ -1462,6 +1479,19 @@ bool ProtoStreamObjectWriter::ValidOneof(const google::protobuf::Field& field, return true; } +bool ProtoStreamObjectWriter::ValidMapKey(StringPiece unnormalized_name) { + if (element_ == NULL) return true; + + if (!element_->InsertMapKeyIfNotPresent(unnormalized_name)) { + InvalidName( + unnormalized_name, + StrCat("Repeated map key: '", unnormalized_name, "' is already set.")); + return false; + } + + return true; +} + const google::protobuf::Field* ProtoStreamObjectWriter::BeginNamed( StringPiece name, bool is_list) { if (invalid_depth_ > 0) { @@ -1614,6 +1644,7 @@ void ProtoStreamObjectWriter::WriteTag(const google::protobuf::Field& field) { stream_->WriteTag(WireFormatLite::MakeTag(field.number(), wire_type)); } + } // namespace converter } // namespace util } // namespace protobuf diff --git a/src/google/protobuf/util/internal/protostream_objectwriter.h b/src/google/protobuf/util/internal/protostream_objectwriter.h index 8f49120be8..19f747fe47 100644 --- a/src/google/protobuf/util/internal/protostream_objectwriter.h +++ b/src/google/protobuf/util/internal/protostream_objectwriter.h @@ -259,6 +259,13 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public StructuredObjectWriter // generate an error. void TakeOneofIndex(int32 index); + // Inserts map key into hash set if and only if the key did NOT already + // exist in hash set. + // The hash set (map_keys_) is ONLY used to keep track of map keys. + // Return true if insert successfully; returns false if the map key was + // already present. + bool InsertMapKeyIfNotPresent(StringPiece map_key); + private: // Used for access to variables of the enclosing instance of // ProtoStreamObjectWriter. @@ -296,6 +303,10 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public StructuredObjectWriter // incoming messages so no more than one oneof is set. hash_set oneof_indices_; + // Set of map keys already seen for the type_. Used to validate incoming + // messages so no map key appears more than once. + hash_set map_keys_; + GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(ProtoElement); }; @@ -378,6 +389,7 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public StructuredObjectWriter // Helper method to write proto tags based on the given field. void WriteTag(const google::protobuf::Field& field); + // Helper function to render primitive data types in DataPiece. void RenderSimpleDataPiece(const google::protobuf::Field& field, const google::protobuf::Type& type, @@ -424,6 +436,14 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public StructuredObjectWriter bool ValidOneof(const google::protobuf::Field& field, StringPiece unnormalized_name); + // Returns true if the map key for type_ is not duplicated key. + // If map key is duplicated key, this function returns false. + // Note that caller should make sure that the current proto element (element_) + // is of element type MAP or STRUCT_MAP. + // It also calls the appropriate error callback and unnormalzied_name is used + // for error string. + bool ValidMapKey(StringPiece unnormalized_name); + // Variables for describing the structure of the input tree: // master_type_: descriptor for the whole protobuf message. // typeinfo_ : the TypeInfo object to lookup types. diff --git a/src/google/protobuf/util/internal/protostream_objectwriter_test.cc b/src/google/protobuf/util/internal/protostream_objectwriter_test.cc index 96e5ccfbf8..941fa71a66 100644 --- a/src/google/protobuf/util/internal/protostream_objectwriter_test.cc +++ b/src/google/protobuf/util/internal/protostream_objectwriter_test.cc @@ -45,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -139,7 +140,9 @@ class BaseProtoStreamObjectWriterTest google::protobuf::scoped_ptr message(expected.New()); message->ParsePartialFromIstream(&istream); - EXPECT_EQ(expected.DebugString(), message->DebugString()); + if (!MessageDifferencer::Equivalent(expected, *message)) { + EXPECT_EQ(expected.DebugString(), message->DebugString()); + } } void CheckOutput(const Message& expected) { CheckOutput(expected, -1); } @@ -851,15 +854,19 @@ class ProtoStreamObjectWriterTimestampDurationTest } }; +INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest, + ProtoStreamObjectWriterTimestampDurationTest, + ::testing::Values( + testing::USE_TYPE_RESOLVER)); + TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidTimestampError1) { TimestampDuration timestamp; EXPECT_CALL( listener_, - InvalidValue( - _, StringPiece("type.googleapis.com/google.protobuf.Timestamp"), - StringPiece("Field 'ts', Illegal timestamp format; timestamps " - "must end with 'Z'"))); + InvalidValue(_, + StringPiece("type.googleapis.com/google.protobuf.Timestamp"), + StringPiece("Field 'ts', Invalid time format: "))); ow_->StartObject("")->RenderString("ts", "")->EndObject(); CheckOutput(timestamp); @@ -870,10 +877,9 @@ TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidTimestampError2) { EXPECT_CALL( listener_, - InvalidValue( - _, StringPiece("type.googleapis.com/google.protobuf.Timestamp"), - StringPiece( - "Field 'ts', Invalid time format: Failed to parse input"))); + InvalidValue(_, + StringPiece("type.googleapis.com/google.protobuf.Timestamp"), + StringPiece("Field 'ts', Invalid time format: Z"))); ow_->StartObject("")->RenderString("ts", "Z")->EndObject(); CheckOutput(timestamp); @@ -884,10 +890,10 @@ TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidTimestampError3) { EXPECT_CALL( listener_, - InvalidValue( - _, StringPiece("type.googleapis.com/google.protobuf.Timestamp"), - StringPiece("Field 'ts', Invalid time format, failed to parse nano " - "seconds"))); + InvalidValue(_, + StringPiece("type.googleapis.com/google.protobuf.Timestamp"), + StringPiece("Field 'ts', Invalid time format: " + "1970-01-01T00:00:00.ABZ"))); ow_->StartObject("") ->RenderString("ts", "1970-01-01T00:00:00.ABZ") @@ -902,7 +908,8 @@ TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidTimestampError4) { listener_, InvalidValue(_, StringPiece("type.googleapis.com/google.protobuf.Timestamp"), - StringPiece("Field 'ts', Timestamp value exceeds limits"))); + StringPiece("Field 'ts', Invalid time format: " + "-8032-10-18T00:00:00.000Z"))); ow_->StartObject("") ->RenderString("ts", "-8032-10-18T00:00:00.000Z") @@ -1008,6 +1015,11 @@ class ProtoStreamObjectWriterStructTest } }; +INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest, + ProtoStreamObjectWriterStructTest, + ::testing::Values( + testing::USE_TYPE_RESOLVER)); + // TODO(skarvaje): Write tests for failure cases. TEST_P(ProtoStreamObjectWriterStructTest, StructRenderSuccess) { StructType struct_type; @@ -1046,12 +1058,62 @@ TEST_P(ProtoStreamObjectWriterStructTest, StructInvalidInputFailure) { CheckOutput(struct_type); } +TEST_P(ProtoStreamObjectWriterStructTest, SimpleRepeatedStructMapKeyTest) { + EXPECT_CALL( + listener_, + InvalidName(_, StringPiece("k1"), + StringPiece("Repeated map key: 'k1' is already set."))); + ow_->StartObject("") + ->StartObject("object") + ->RenderString("k1", "v1") + ->RenderString("k1", "v2") + ->EndObject() + ->EndObject(); +} + +TEST_P(ProtoStreamObjectWriterStructTest, RepeatedStructMapListKeyTest) { + EXPECT_CALL( + listener_, + InvalidName(_, StringPiece("k1"), + StringPiece("Repeated map key: 'k1' is already set."))); + ow_->StartObject("") + ->StartObject("object") + ->RenderString("k1", "v1") + ->StartList("k1") + ->RenderString("", "v2") + ->EndList() + ->EndObject() + ->EndObject(); +} + +TEST_P(ProtoStreamObjectWriterStructTest, RepeatedStructMapObjectKeyTest) { + EXPECT_CALL( + listener_, + InvalidName(_, StringPiece("k1"), + StringPiece("Repeated map key: 'k1' is already set."))); + ow_->StartObject("") + ->StartObject("object") + ->StartObject("k1") + ->RenderString("sub_k1", "v1") + ->EndObject() + ->StartObject("k1") + ->RenderString("sub_k2", "v2") + ->EndObject() + ->EndObject() + ->EndObject(); +} + class ProtoStreamObjectWriterMapTest : public BaseProtoStreamObjectWriterTest { protected: ProtoStreamObjectWriterMapTest() : BaseProtoStreamObjectWriterTest(MapIn::descriptor()) {} }; +INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest, + ProtoStreamObjectWriterMapTest, + ::testing::Values( + testing::USE_TYPE_RESOLVER)); + TEST_P(ProtoStreamObjectWriterMapTest, MapShouldNotAcceptList) { MapIn mm; EXPECT_CALL(listener_, @@ -1066,17 +1128,37 @@ TEST_P(ProtoStreamObjectWriterMapTest, MapShouldNotAcceptList) { CheckOutput(mm); } +TEST_P(ProtoStreamObjectWriterMapTest, RepeatedMapKeyTest) { + EXPECT_CALL( + listener_, + InvalidName(_, StringPiece("k1"), + StringPiece("Repeated map key: 'k1' is already set."))); + ow_->StartObject("") + ->RenderString("other", "test") + ->StartObject("map_input") + ->RenderString("k1", "v1") + ->RenderString("k1", "v2") + ->EndObject() + ->EndObject(); +} + class ProtoStreamObjectWriterAnyTest : public BaseProtoStreamObjectWriterTest { protected: ProtoStreamObjectWriterAnyTest() { vector descriptors; descriptors.push_back(AnyOut::descriptor()); descriptors.push_back(google::protobuf::DoubleValue::descriptor()); + descriptors.push_back(google::protobuf::Timestamp::descriptor()); descriptors.push_back(google::protobuf::Any::descriptor()); ResetTypeInfo(descriptors); } }; +INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest, + ProtoStreamObjectWriterAnyTest, + ::testing::Values( + testing::USE_TYPE_RESOLVER)); + TEST_P(ProtoStreamObjectWriterAnyTest, AnyRenderSuccess) { AnyOut any; google::protobuf::Any* any_type = any.mutable_any(); @@ -1119,8 +1201,6 @@ TEST_P(ProtoStreamObjectWriterAnyTest, RecursiveAny) { ->EndObject() ->EndObject() ->EndObject(); - - CheckOutput(out, 115); } TEST_P(ProtoStreamObjectWriterAnyTest, DoubleRecursiveAny) { @@ -1155,8 +1235,6 @@ TEST_P(ProtoStreamObjectWriterAnyTest, DoubleRecursiveAny) { ->EndObject() ->EndObject() ->EndObject(); - - CheckOutput(out, 159); } TEST_P(ProtoStreamObjectWriterAnyTest, EmptyAnyFromEmptyObject) { @@ -1263,6 +1341,23 @@ TEST_P(ProtoStreamObjectWriterAnyTest, AnyNullInputFails) { CheckOutput(any); } +TEST_P(ProtoStreamObjectWriterAnyTest, AnyWellKnownTypeErrorTest) { + EXPECT_CALL(listener_, InvalidValue(_, StringPiece("Any"), + StringPiece("Invalid time format: "))); + + AnyOut any; + google::protobuf::Any* any_type = any.mutable_any(); + any_type->set_type_url("type.googleapis.com/google.protobuf.Timestamp"); + + ow_->StartObject("") + ->StartObject("any") + ->RenderString("@type", "type.googleapis.com/google.protobuf.Timestamp") + ->RenderString("value", "") + ->EndObject() + ->EndObject(); + CheckOutput(any); +} + class ProtoStreamObjectWriterFieldMaskTest : public BaseProtoStreamObjectWriterTest { protected: @@ -1274,6 +1369,11 @@ class ProtoStreamObjectWriterFieldMaskTest } }; +INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest, + ProtoStreamObjectWriterFieldMaskTest, + ::testing::Values( + testing::USE_TYPE_RESOLVER)); + TEST_P(ProtoStreamObjectWriterFieldMaskTest, SimpleFieldMaskTest) { FieldMaskTest expected; expected.set_id("1"); @@ -1683,6 +1783,7 @@ TEST_P(ProtoStreamObjectWriterOneOfsTest, ow_->RenderString("strData", "blah"); ow_->RenderInt32("intData", 123); ow_->EndObject(); + ow_->EndObject(); } } // namespace converter diff --git a/src/google/protobuf/util/internal/utility.cc b/src/google/protobuf/util/internal/utility.cc index 5d7dcc8756..61899c24ea 100644 --- a/src/google/protobuf/util/internal/utility.cc +++ b/src/google/protobuf/util/internal/utility.cc @@ -298,16 +298,21 @@ bool IsMap(const google::protobuf::Field& field, "google.protobuf.MessageOptions.map_entry", false)); } +bool IsMessageSetWireFormat(const google::protobuf::Type& type) { + return GetBoolOptionOrDefault( + type.options(), "google.protobuf.MessageOptions.message_set_wire_format", false); +} + string DoubleAsString(double value) { - if (google::protobuf::MathLimits::IsPosInf(value)) return "Infinity"; - if (google::protobuf::MathLimits::IsNegInf(value)) return "-Infinity"; - if (google::protobuf::MathLimits::IsNaN(value)) return "NaN"; + if (MathLimits::IsPosInf(value)) return "Infinity"; + if (MathLimits::IsNegInf(value)) return "-Infinity"; + if (MathLimits::IsNaN(value)) return "NaN"; return SimpleDtoa(value); } string FloatAsString(float value) { - if (google::protobuf::MathLimits::IsFinite(value)) return SimpleFtoa(value); + if (MathLimits::IsFinite(value)) return SimpleFtoa(value); return DoubleAsString(value); } @@ -318,7 +323,7 @@ bool SafeStrToFloat(StringPiece str, float *value) { } *value = static_cast(double_value); - if (google::protobuf::MathLimits::IsInf(*value)) { + if (MathLimits::IsInf(*value)) { return false; } return true; diff --git a/src/google/protobuf/util/internal/utility.h b/src/google/protobuf/util/internal/utility.h index 87f7602a9c..5ba97bd231 100644 --- a/src/google/protobuf/util/internal/utility.h +++ b/src/google/protobuf/util/internal/utility.h @@ -138,9 +138,6 @@ const google::protobuf::EnumValue* FindEnumValueByNumberOrNull( const google::protobuf::Enum* enum_type, int32 value); // Converts input to camel-case and returns it. -// Tests are in wrappers/translator/snake2camel_objectwriter_test.cc -// TODO(skarvaje): Isolate tests for this function and put them in -// utility_test.cc LIBPROTOBUF_EXPORT string ToCamelCase(const StringPiece input); // Converts input to snake_case and returns it. @@ -157,6 +154,9 @@ LIBPROTOBUF_EXPORT bool IsValidBoolString(const string& bool_string); LIBPROTOBUF_EXPORT bool IsMap(const google::protobuf::Field& field, const google::protobuf::Type& type); +// Returns true if the given type has special MessageSet wire format. +bool IsMessageSetWireFormat(const google::protobuf::Type& type); + // Infinity/NaN-aware conversion to string. LIBPROTOBUF_EXPORT string DoubleAsString(double value); LIBPROTOBUF_EXPORT string FloatAsString(float value); diff --git a/src/google/protobuf/util/json_format_proto3.proto b/src/google/protobuf/util/json_format_proto3.proto index 7a2828681e..e81376778c 100644 --- a/src/google/protobuf/util/json_format_proto3.proto +++ b/src/google/protobuf/util/json_format_proto3.proto @@ -100,6 +100,16 @@ message TestMap { map string_map = 6; } +message TestNestedMap { + map bool_map = 1; + map int32_map = 2; + map int64_map = 3; + map uint32_map = 4; + map uint64_map = 5; + map string_map = 6; + map map_map = 7; +} + message TestWrapper { google.protobuf.BoolValue bool_value = 1; google.protobuf.Int32Value int32_value = 2; diff --git a/src/google/protobuf/util/json_util.cc b/src/google/protobuf/util/json_util.cc index 6cd40fd5ac..c3b8d502e3 100644 --- a/src/google/protobuf/util/json_util.cc +++ b/src/google/protobuf/util/json_util.cc @@ -34,7 +34,6 @@ #include #include #include -#include #include #include #include @@ -83,13 +82,12 @@ util::Status BinaryToJsonStream(TypeResolver* resolver, io::CodedOutputStream out_stream(json_output); converter::JsonObjectWriter json_writer(options.add_whitespace ? " " : "", &out_stream); - converter::Snake2CamelObjectWriter snake2camel_writer(&json_writer); if (options.always_print_primitive_fields) { converter::DefaultValueObjectWriter default_value_writer( - resolver, type, &snake2camel_writer); + resolver, type, &json_writer); return proto_source.WriteTo(&default_value_writer); } else { - return proto_source.WriteTo(&snake2camel_writer); + return proto_source.WriteTo(&json_writer); } } diff --git a/src/google/protobuf/util/message_differencer.cc b/src/google/protobuf/util/message_differencer.cc index d709da57d8..9a2689738b 100644 --- a/src/google/protobuf/util/message_differencer.cc +++ b/src/google/protobuf/util/message_differencer.cc @@ -946,6 +946,18 @@ bool MessageDifferencer::IsIgnored( return false; } +bool MessageDifferencer::IsUnknownFieldIgnored( + const Message& message1, const Message& message2, + const SpecificField& field, const vector& parent_fields) { + for (int i = 0; i < ignore_criteria_.size(); ++i) { + if (ignore_criteria_[i]->IsUnknownFieldIgnored(message1, message2, field, + parent_fields)) { + return true; + } + } + return false; +} + const MessageDifferencer::MapKeyComparator* MessageDifferencer ::GetMapKeyComparator(const FieldDescriptor* field) { if (!field->is_repeated()) return NULL; @@ -1127,15 +1139,6 @@ bool MessageDifferencer::CompareUnknownFields( continue; } - if (change_type == ADDITION || change_type == DELETION || - change_type == MODIFICATION) { - if (reporter_ == NULL) { - // We found a difference and we have no reproter. - return false; - } - is_different = true; - } - // Build the SpecificField. This is slightly complicated. SpecificField specific_field; specific_field.unknown_field_number = focus_field->number(); @@ -1160,6 +1163,25 @@ bool MessageDifferencer::CompareUnknownFields( specific_field.new_index = index2 - current_repeated_start2; } + if (IsUnknownFieldIgnored(message1, message2, specific_field, + *parent_field)) { + if (reporter_ != NULL) { + parent_field->push_back(specific_field); + reporter_->ReportUnknownFieldIgnored(message1, message2, *parent_field); + parent_field->pop_back(); + } + return true; + } + + if (change_type == ADDITION || change_type == DELETION || + change_type == MODIFICATION) { + if (reporter_ == NULL) { + // We found a difference and we have no reproter. + return false; + } + is_different = true; + } + parent_field->push_back(specific_field); switch (change_type) { @@ -1625,6 +1647,18 @@ void MessageDifferencer::StreamReporter::ReportIgnored( printer_->Print("\n"); // Print for newlines. } +void MessageDifferencer::StreamReporter::ReportUnknownFieldIgnored( + const Message& message1, const Message& message2, + const vector& field_path) { + printer_->Print("ignored: "); + PrintPath(field_path, true); + if (CheckPathChanged(field_path)) { + printer_->Print(" -> "); + PrintPath(field_path, false); + } + printer_->Print("\n"); // Print for newlines. +} + } // namespace util } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/util/message_differencer.h b/src/google/protobuf/util/message_differencer.h index e002a0f3a1..34c173dbf0 100644 --- a/src/google/protobuf/util/message_differencer.h +++ b/src/google/protobuf/util/message_differencer.h @@ -278,6 +278,13 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { const Message& message2, const vector& field_path) { } + // Report that an unkown field is ignored. (see comment above). + // Note this is a different function since the last SpecificField in field + // path has a null field. This could break existing Reporter. + virtual void ReportUnknownFieldIgnored( + const Message& message1, const Message& message2, + const vector& field_path) {} + private: GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Reporter); }; @@ -317,6 +324,16 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { const Message& message2, const FieldDescriptor* field, const vector& parent_fields) = 0; + + // Returns true if the unknown field should be ignored. + // Note: This will be called for unknown fields as well in which case + // field.field will be null. + virtual bool IsUnknownFieldIgnored( + const Message& message1, const Message& message2, + const SpecificField& field, + const vector& parent_fields) { + return false; + } }; // To add a Reporter, construct default here, then use ReportDifferencesTo or @@ -583,6 +600,10 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { const Message& message2, const vector& field_path); + virtual void ReportUnknownFieldIgnored( + const Message& message1, const Message& message2, + const vector& field_path); + protected: // Prints the specified path of fields to the buffer. virtual void PrintPath(const vector& field_path, @@ -722,6 +743,12 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { const FieldDescriptor* field, const vector& parent_fields); + // Returns true if this unknown field is to be ignored when this + // MessageDifferencer compares messages. + bool IsUnknownFieldIgnored(const Message& message1, const Message& message2, + const SpecificField& field, + const vector& parent_fields); + // Returns MapKeyComparator* when this field has been configured to // be treated as a map. If not, returns NULL. const MapKeyComparator* GetMapKeyComparator(const FieldDescriptor* field); diff --git a/src/google/protobuf/util/time_util.h b/src/google/protobuf/util/time_util.h index 58dbf8e6e4..1bac089707 100644 --- a/src/google/protobuf/util/time_util.h +++ b/src/google/protobuf/util/time_util.h @@ -41,7 +41,6 @@ #endif #include -#include #include namespace google { @@ -243,10 +242,8 @@ inline ostream& operator<<(ostream& out, const Duration& d) { // Overloaded operators for Timestamp // // Assignement operators. -LIBPROTOBUF_EXPORT -Timestamp& operator+=(Timestamp& t, const Duration& d); // NOLINT -LIBPROTOBUF_EXPORT -Timestamp& operator-=(Timestamp& t, const Duration& d); // NOLINT +LIBPROTOBUF_EXPORT Timestamp& operator+=(Timestamp& t, const Duration& d); // NOLINT +LIBPROTOBUF_EXPORT Timestamp& operator-=(Timestamp& t, const Duration& d); // NOLINT // Relational operators. inline bool operator<(const Timestamp& t1, const Timestamp& t2) { if (t1.seconds() == t2.seconds()) { diff --git a/src/google/protobuf/util/type_resolver_util.cc b/src/google/protobuf/util/type_resolver_util.cc index 908634ebbd..a09969548d 100644 --- a/src/google/protobuf/util/type_resolver_util.cc +++ b/src/google/protobuf/util/type_resolver_util.cc @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -53,8 +54,7 @@ using util::Status; using util::error::INVALID_ARGUMENT; using util::error::NOT_FOUND; -bool SplitTypeUrl(const string& type_url, - string* url_prefix, +bool SplitTypeUrl(const string& type_url, string* url_prefix, string* message_name) { size_t pos = type_url.find_last_of("/"); if (pos == string::npos) { @@ -65,60 +65,19 @@ bool SplitTypeUrl(const string& type_url, return true; } -// This code is originally defined in -// //google/protobuf/util/converter/utility.h. Copied here due to component -// dependency. -// TODO(xiaofeng): Remove this when converter code is in components. -string ToCamelCase(const StringPiece input) { - bool capitalize_next = false; - bool was_cap = true; - bool is_cap = false; - bool first_word = true; - string result; - result.reserve(input.size()); - - for (size_t i = 0; i < input.size(); ++i, was_cap = is_cap) { - is_cap = ascii_isupper(input[i]); - if (input[i] == '_') { - capitalize_next = true; - if (!result.empty()) first_word = false; - continue; - } else if (first_word) { - // Consider when the current character B is capitalized, - // first word ends when: - // 1) following a lowercase: "...aB..." - // 2) followed by a lowercase: "...ABc..." - if (!result.empty() && is_cap && - (!was_cap || (i + 1 < input.size() && ascii_islower(input[i + 1])))) { - first_word = false; - } else { - result.push_back(ascii_tolower(input[i])); - continue; - } - } else if (capitalize_next) { - capitalize_next = false; - if (ascii_islower(input[i])) { - result.push_back(ascii_toupper(input[i])); - continue; - } - } - result.push_back(input[i]); - } - return result; -} - class DescriptorPoolTypeResolver : public TypeResolver { public: DescriptorPoolTypeResolver(const string& url_prefix, const DescriptorPool* pool) - : url_prefix_(url_prefix), pool_(pool) { - } + : url_prefix_(url_prefix), pool_(pool) {} Status ResolveMessageType(const string& type_url, Type* type) { string url_prefix, message_name; if (!SplitTypeUrl(type_url, &url_prefix, &message_name) || url_prefix != url_prefix_) { - return Status(INVALID_ARGUMENT, "Failed to parse type url: " + type_url); + return Status(INVALID_ARGUMENT, + StrCat("Invalid type URL, type URLs must be of the form '", + url_prefix_, "/', got: ", type_url)); } if (url_prefix != url_prefix_) { return Status(INVALID_ARGUMENT, @@ -126,7 +85,8 @@ class DescriptorPoolTypeResolver : public TypeResolver { } const Descriptor* descriptor = pool_->FindMessageTypeByName(message_name); if (descriptor == NULL) { - return Status(NOT_FOUND, "Cannot found the type: " + message_name); + return Status(NOT_FOUND, + "Invalid type URL, unknown type: " + message_name); } ConvertDescriptor(descriptor, type); return Status(); @@ -136,7 +96,9 @@ class DescriptorPoolTypeResolver : public TypeResolver { string url_prefix, type_name; if (!SplitTypeUrl(type_url, &url_prefix, &type_name) || url_prefix != url_prefix_) { - return Status(INVALID_ARGUMENT, "Failed to parse type url: " + type_url); + return Status(INVALID_ARGUMENT, + StrCat("Invalid type URL, type URLs must be of the form '", + url_prefix_, "/', got: ", type_url)); } if (url_prefix != url_prefix_) { return Status(INVALID_ARGUMENT, @@ -144,7 +106,7 @@ class DescriptorPoolTypeResolver : public TypeResolver { } const EnumDescriptor* descriptor = pool_->FindEnumTypeByName(type_name); if (descriptor == NULL) { - return Status(NOT_FOUND, "Cannot found the type: " + type_name); + return Status(NOT_FOUND, "Invalid type URL, unknown type: " + type_name); } ConvertEnumDescriptor(descriptor, enum_type); return Status(); @@ -197,7 +159,7 @@ class DescriptorPoolTypeResolver : public TypeResolver { } field->set_number(descriptor->number()); field->set_name(descriptor->name()); - field->set_json_name(ToCamelCase(descriptor->name())); + field->set_json_name(converter::ToCamelCase(descriptor->name())); if (descriptor->type() == FieldDescriptor::TYPE_MESSAGE) { field->set_type_url(GetTypeUrl(descriptor->message_type())); } else if (descriptor->type() == FieldDescriptor::TYPE_ENUM) { @@ -221,8 +183,7 @@ class DescriptorPoolTypeResolver : public TypeResolver { descriptor->file()->name()); for (int i = 0; i < descriptor->value_count(); ++i) { const EnumValueDescriptor* value_descriptor = descriptor->value(i); - EnumValue* value = - enum_type->mutable_enumvalue()->Add(); + EnumValue* value = enum_type->mutable_enumvalue()->Add(); value->set_name(value_descriptor->name()); value->set_number(value_descriptor->number()); @@ -245,8 +206,8 @@ class DescriptorPoolTypeResolver : public TypeResolver { } // namespace -TypeResolver* NewTypeResolverForDescriptorPool( - const string& url_prefix, const DescriptorPool* pool) { +TypeResolver* NewTypeResolverForDescriptorPool(const string& url_prefix, + const DescriptorPool* pool) { return new DescriptorPoolTypeResolver(url_prefix, pool); }