diff --git a/cmake/libprotobuf-lite.cmake b/cmake/libprotobuf-lite.cmake index 19f4f25d56..8c7501f31a 100644 --- a/cmake/libprotobuf-lite.cmake +++ b/cmake/libprotobuf-lite.cmake @@ -39,6 +39,7 @@ set(libprotobuf_lite_includes ${protobuf_SOURCE_DIR}/src/google/protobuf/arena_impl.h ${protobuf_SOURCE_DIR}/src/google/protobuf/arenastring.h ${protobuf_SOURCE_DIR}/src/google/protobuf/arenaz_sampler.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/endian.h ${protobuf_SOURCE_DIR}/src/google/protobuf/explicitly_constructed.h ${protobuf_SOURCE_DIR}/src/google/protobuf/extension_set.h ${protobuf_SOURCE_DIR}/src/google/protobuf/extension_set_inl.h diff --git a/cmake/libprotobuf.cmake b/cmake/libprotobuf.cmake index be5b229700..abb2f79eba 100644 --- a/cmake/libprotobuf.cmake +++ b/cmake/libprotobuf.cmake @@ -78,6 +78,7 @@ set(libprotobuf_includes ${protobuf_SOURCE_DIR}/src/google/protobuf/message.h ${protobuf_SOURCE_DIR}/src/google/protobuf/metadata.h ${protobuf_SOURCE_DIR}/src/google/protobuf/reflection.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/reflection_internal.h ${protobuf_SOURCE_DIR}/src/google/protobuf/reflection_ops.h ${protobuf_SOURCE_DIR}/src/google/protobuf/service.h ${protobuf_SOURCE_DIR}/src/google/protobuf/source_context.pb.h diff --git a/cmake/libprotoc.cmake b/cmake/libprotoc.cmake index 607c2748e3..be9db8273d 100644 --- a/cmake/libprotoc.cmake +++ b/cmake/libprotoc.cmake @@ -97,17 +97,14 @@ set(libprotoc_headers ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/csharp/csharp_generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/csharp/csharp_names.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/csharp/csharp_options.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/importer.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/kotlin_generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/names.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/js/js_generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/objectivec_generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/objectivec_helpers.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/parser.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/php/php_generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/plugin.h - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/plugin.pb.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/python/generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/python/pyi_generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/ruby/ruby_generator.h diff --git a/cmake/tests.cmake b/cmake/tests.cmake index ceaf2964b4..ca34152f69 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -129,21 +129,23 @@ set(common_lite_test_files ${protobuf_SOURCE_DIR}/src/google/protobuf/test_util_lite.cc ) -add_library(protobuf-lite-test-common ${protobuf_SHARED_OR_STATIC} +add_library(protobuf-lite-test-common STATIC ${common_lite_test_files} ${lite_test_proto_files}) +target_link_libraries(protobuf-lite-test-common libprotobuf-lite GTest::gmock) set(common_test_files ${common_lite_test_files} + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/mock_code_generator.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/map_test_util.inc ${protobuf_SOURCE_DIR}/src/google/protobuf/reflection_tester.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/test_util.cc - ${protobuf_SOURCE_DIR}/src/google/protobuf/test_util.inc ${protobuf_SOURCE_DIR}/src/google/protobuf/testing/file.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/testing/googletest.cc ) -add_library(protobuf-test-common ${protobuf_SHARED_OR_STATIC} +add_library(protobuf-test-common STATIC ${common_test_files} ${tests_proto_files}) +target_link_libraries(protobuf-test-common libprotobuf GTest::gmock) set(tests_files ${protobuf_SOURCE_DIR}/src/google/protobuf/any_test.cc @@ -151,6 +153,7 @@ set(tests_files ${protobuf_SOURCE_DIR}/src/google/protobuf/arenastring_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/arenaz_sampler_test.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/annotation_test_util.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/annotation_test_util.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/command_line_interface_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/cpp/bootstrap_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/cpp/message_size_unittest.cc @@ -164,7 +167,6 @@ set(tests_files ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/importer_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/doc_comment_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/plugin_unittest.cc - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/mock_code_generator.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/objectivec_helpers_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/parser_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/python/plugin_unittest.cc diff --git a/java/core/src/main/java/com/google/protobuf/MapEntry.java b/java/core/src/main/java/com/google/protobuf/MapEntry.java index d528e1a216..8890f225f7 100644 --- a/java/core/src/main/java/com/google/protobuf/MapEntry.java +++ b/java/core/src/main/java/com/google/protobuf/MapEntry.java @@ -336,13 +336,17 @@ public final class MapEntry extends AbstractMessage { @Override public Builder setField(FieldDescriptor field, Object value) { checkFieldDescriptor(field); + if (value == null) { + throw new NullPointerException(field.getFullName() + " is null"); + } + if (field.getNumber() == 1) { setKey((K) value); } else { if (field.getType() == FieldDescriptor.Type.ENUM) { value = ((EnumValueDescriptor) value).getNumber(); } else if (field.getType() == FieldDescriptor.Type.MESSAGE) { - if (value != null && !metadata.defaultValue.getClass().isInstance(value)) { + if (!metadata.defaultValue.getClass().isInstance(value)) { // The value is not the exact right message type. However, if it // is an alternative implementation of the same type -- e.g. a // DynamicMessage -- we should accept it. In this case we can make diff --git a/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java b/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java index 5c482d62da..45b5a6b306 100644 --- a/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java +++ b/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java @@ -62,7 +62,7 @@ public final class UnknownFieldSet implements MessageLite { /** * Construct an {@code UnknownFieldSet} around the given map. */ - UnknownFieldSet(TreeMap fields) { + private UnknownFieldSet(TreeMap fields) { this.fields = fields; } diff --git a/java/core/src/main/java/com/google/protobuf/Utf8.java b/java/core/src/main/java/com/google/protobuf/Utf8.java index d69868100c..a178d2a931 100644 --- a/java/core/src/main/java/com/google/protobuf/Utf8.java +++ b/java/core/src/main/java/com/google/protobuf/Utf8.java @@ -1605,6 +1605,18 @@ final class Utf8 { return 0; } + // Read bytes until 8-byte aligned so that we can read longs in the loop below. + // Byte arrays are already either 8 or 16-byte aligned, so we just need to make sure that + // the index (relative to the start of the array) is also 8-byte aligned. We do this by + // ANDing the index with 7 to determine the number of bytes that need to be read before + // we're 8-byte aligned. + final int unaligned = 8 - ((int) offset & 7); + for (int j = unaligned; j > 0; j--) { + if (UnsafeUtil.getByte(bytes, offset++) < 0) { + return unaligned - j; + } + } + int i; for (i = 0; i + 8 <= maxChars; i += 8) { if ((UnsafeUtil.getLong(bytes, UnsafeUtil.BYTE_ARRAY_BASE_OFFSET + offset) diff --git a/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java b/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java index 55864afa72..0188f2f940 100644 --- a/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java +++ b/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java @@ -59,6 +59,7 @@ import protobuf_unittest.UnittestProto.TestAllExtensions; import protobuf_unittest.UnittestProto.TestAllTypes; import protobuf_unittest.UnittestProto.TestAllTypes.NestedMessage; import protobuf_unittest.UnittestProto.TestAllTypesOrBuilder; +import protobuf_unittest.UnittestProto.TestChildExtension; import protobuf_unittest.UnittestProto.TestExtremeDefaultValues; import protobuf_unittest.UnittestProto.TestOneof2; import protobuf_unittest.UnittestProto.TestPackedTypes; @@ -2023,4 +2024,5 @@ public class GeneratedMessageTest { assertThat(builder.getRepeatedField(REPEATED_NESTED_MESSAGE_EXTENSION, 0)) .isEqualTo(NestedMessage.newBuilder().setBb(100).build()); } + } diff --git a/java/core/src/test/java/com/google/protobuf/MapTest.java b/java/core/src/test/java/com/google/protobuf/MapTest.java index 7884497ee4..1bb7166718 100644 --- a/java/core/src/test/java/com/google/protobuf/MapTest.java +++ b/java/core/src/test/java/com/google/protobuf/MapTest.java @@ -40,6 +40,7 @@ import com.google.protobuf.Descriptors.EnumDescriptor; import com.google.protobuf.Descriptors.EnumValueDescriptor; import com.google.protobuf.Descriptors.FieldDescriptor; import map_test.MapTestProto.BizarroTestMap; +import map_test.MapTestProto.MapContainer; import map_test.MapTestProto.ReservedAsMapField; import map_test.MapTestProto.ReservedAsMapFieldWithEnumValue; import map_test.MapTestProto.TestMap; @@ -1586,4 +1587,19 @@ public class MapTest { assertThat(builder.build().toByteArray()).isEqualTo(new byte[0]); } + + @Test + // https://github.com/protocolbuffers/protobuf/issues/9785 + public void testContainer() { + FieldDescriptor field = MapContainer.getDescriptor().findFieldByName("my_map"); + Descriptor entryDescriptor = field.getMessageType(); + FieldDescriptor valueDescriptor = entryDescriptor.findFieldByName("value"); + Message.Builder builder = MapContainer.newBuilder().newBuilderForField(field); + try { + builder.setField(valueDescriptor, null); + fail("Allowed null field value"); + } catch (NullPointerException expected) { + assertThat(expected).hasMessageThat().isNotNull(); + } + } } diff --git a/java/core/src/test/java/com/google/protobuf/TestUtil.java b/java/core/src/test/java/com/google/protobuf/TestUtil.java index 0b1852e3f0..46a4d17d61 100644 --- a/java/core/src/test/java/com/google/protobuf/TestUtil.java +++ b/java/core/src/test/java/com/google/protobuf/TestUtil.java @@ -80,6 +80,7 @@ import static com.google.protobuf.UnittestLite.optionalStringExtensionLite; import static com.google.protobuf.UnittestLite.optionalStringPieceExtensionLite; import static com.google.protobuf.UnittestLite.optionalUint32ExtensionLite; import static com.google.protobuf.UnittestLite.optionalUint64ExtensionLite; +import static com.google.protobuf.UnittestLite.optionalUnverifiedLazyMessageExtensionLite; import static com.google.protobuf.UnittestLite.packedBoolExtensionLite; import static com.google.protobuf.UnittestLite.packedDoubleExtensionLite; import static com.google.protobuf.UnittestLite.packedEnumExtensionLite; @@ -169,6 +170,7 @@ import static protobuf_unittest.UnittestProto.optionalStringExtension; import static protobuf_unittest.UnittestProto.optionalStringPieceExtension; import static protobuf_unittest.UnittestProto.optionalUint32Extension; import static protobuf_unittest.UnittestProto.optionalUint64Extension; +import static protobuf_unittest.UnittestProto.optionalUnverifiedLazyMessageExtension; import static protobuf_unittest.UnittestProto.packedBoolExtension; import static protobuf_unittest.UnittestProto.packedDoubleExtension; import static protobuf_unittest.UnittestProto.packedEnumExtension; @@ -343,6 +345,8 @@ public final class TestUtil { message.setOptionalImportMessage(ImportMessage.newBuilder().setD(120).build()); message.setOptionalPublicImportMessage(PublicImportMessage.newBuilder().setE(126).build()); message.setOptionalLazyMessage(TestAllTypes.NestedMessage.newBuilder().setBb(127).build()); + message.setOptionalUnverifiedLazyMessage( + TestAllTypes.NestedMessage.newBuilder().setBb(128).build()); message.setOptionalNestedEnum(TestAllTypes.NestedEnum.BAZ); message.setOptionalForeignEnum(ForeignEnum.FOREIGN_BAZ); @@ -1240,6 +1244,9 @@ public final class TestUtil { optionalPublicImportMessageExtension, PublicImportMessage.newBuilder().setE(126).build()); message.setExtension( optionalLazyMessageExtension, TestAllTypes.NestedMessage.newBuilder().setBb(127).build()); + message.setExtension( + optionalUnverifiedLazyMessageExtension, + TestAllTypes.NestedMessage.newBuilder().setBb(128).build()); message.setExtension(optionalNestedEnumExtension, TestAllTypes.NestedEnum.BAZ); message.setExtension(optionalForeignEnumExtension, ForeignEnum.FOREIGN_BAZ); @@ -1459,6 +1466,8 @@ public final class TestUtil { assertEqualsExactType(120, message.getExtension(optionalImportMessageExtension).getD()); assertEqualsExactType(126, message.getExtension(optionalPublicImportMessageExtension).getE()); assertEqualsExactType(127, message.getExtension(optionalLazyMessageExtension).getBb()); + assertEqualsExactType( + 128, message.getExtension(optionalUnverifiedLazyMessageExtension).getBb()); assertEqualsExactType( TestAllTypes.NestedEnum.BAZ, message.getExtension(optionalNestedEnumExtension)); @@ -2051,6 +2060,8 @@ public final class TestUtil { assertEqualsExactType( 126, message.getExtension(optionalPublicImportMessageExtensionLite).getE()); assertEqualsExactType(127, message.getExtension(optionalLazyMessageExtensionLite).getBb()); + assertEqualsExactType( + 128, message.getExtension(optionalUnverifiedLazyMessageExtensionLite).getBb()); assertEqualsExactType( TestAllTypesLite.NestedEnum.BAZ, message.getExtension(optionalNestedEnumExtensionLite)); @@ -2244,6 +2255,7 @@ public final class TestUtil { Assert.assertFalse(message.hasExtension(optionalImportMessageExtensionLite)); Assert.assertFalse(message.hasExtension(optionalPublicImportMessageExtensionLite)); Assert.assertFalse(message.hasExtension(optionalLazyMessageExtensionLite)); + Assert.assertFalse(message.hasExtension(optionalUnverifiedLazyMessageExtensionLite)); Assert.assertFalse(message.hasExtension(optionalNestedEnumExtensionLite)); Assert.assertFalse(message.hasExtension(optionalForeignEnumExtensionLite)); @@ -2276,6 +2288,7 @@ public final class TestUtil { Assert.assertFalse(message.getExtension(optionalImportMessageExtensionLite).hasD()); Assert.assertFalse(message.getExtension(optionalPublicImportMessageExtensionLite).hasE()); Assert.assertFalse(message.getExtension(optionalLazyMessageExtensionLite).hasBb()); + Assert.assertFalse(message.getExtension(optionalUnverifiedLazyMessageExtensionLite).hasBb()); assertEqualsExactType(0, message.getExtension(optionalGroupExtensionLite).getA()); assertEqualsExactType(0, message.getExtension(optionalNestedMessageExtensionLite).getBb()); @@ -2283,6 +2296,8 @@ public final class TestUtil { assertEqualsExactType(0, message.getExtension(optionalImportMessageExtensionLite).getD()); assertEqualsExactType(0, message.getExtension(optionalPublicImportMessageExtensionLite).getE()); assertEqualsExactType(0, message.getExtension(optionalLazyMessageExtensionLite).getBb()); + assertEqualsExactType( + 0, message.getExtension(optionalUnverifiedLazyMessageExtensionLite).getBb()); // Enums without defaults are set to the first value in the enum. assertEqualsExactType( @@ -2850,6 +2865,11 @@ public final class TestUtil { message.setField( f("optional_lazy_message"), newBuilderForField(message, f("optional_lazy_message")).setField(nestedB, 127).build()); + message.setField( + f("optional_unverified_lazy_message"), + newBuilderForField(message, f("optional_unverified_lazy_message")) + .setField(nestedB, 128) + .build()); message.setField(f("optional_nested_enum"), nestedBaz); message.setField(f("optional_foreign_enum"), foreignBaz); @@ -3100,6 +3120,9 @@ public final class TestUtil { 126, ((Message) message.getField(f("optional_public_import_message"))).getField(importE)); Assert.assertEquals( 127, ((Message) message.getField(f("optional_lazy_message"))).getField(nestedB)); + Assert.assertEquals( + 128, + ((Message) message.getField(f("optional_unverified_lazy_message"))).getField(nestedB)); Assert.assertEquals(nestedBaz, message.getField(f("optional_nested_enum"))); Assert.assertEquals(foreignBaz, message.getField(f("optional_foreign_enum"))); @@ -3351,6 +3374,8 @@ public final class TestUtil { ((Message) message.getField(f("optional_public_import_message"))).hasField(importE)); Assert.assertFalse( ((Message) message.getField(f("optional_lazy_message"))).hasField(nestedB)); + Assert.assertFalse( + ((Message) message.getField(f("optional_unverified_lazy_message"))).hasField(nestedB)); Assert.assertEquals(0, ((Message) message.getField(f("optionalgroup"))).getField(groupA)); Assert.assertEquals( @@ -3363,6 +3388,8 @@ public final class TestUtil { 0, ((Message) message.getField(f("optional_public_import_message"))).getField(importE)); Assert.assertEquals( 0, ((Message) message.getField(f("optional_lazy_message"))).getField(nestedB)); + Assert.assertEquals( + 0, ((Message) message.getField(f("optional_unverified_lazy_message"))).getField(nestedB)); // Enums without defaults are set to the first value in the enum. Assert.assertEquals(nestedFoo, message.getField(f("optional_nested_enum"))); diff --git a/java/core/src/test/java/com/google/protobuf/TestUtilLite.java b/java/core/src/test/java/com/google/protobuf/TestUtilLite.java index 993dd9defb..dd8c11a3b3 100644 --- a/java/core/src/test/java/com/google/protobuf/TestUtilLite.java +++ b/java/core/src/test/java/com/google/protobuf/TestUtilLite.java @@ -80,6 +80,7 @@ import static com.google.protobuf.UnittestLite.optionalStringExtensionLite; import static com.google.protobuf.UnittestLite.optionalStringPieceExtensionLite; import static com.google.protobuf.UnittestLite.optionalUint32ExtensionLite; import static com.google.protobuf.UnittestLite.optionalUint64ExtensionLite; +import static com.google.protobuf.UnittestLite.optionalUnverifiedLazyMessageExtensionLite; import static com.google.protobuf.UnittestLite.packedBoolExtensionLite; import static com.google.protobuf.UnittestLite.packedDoubleExtensionLite; import static com.google.protobuf.UnittestLite.packedEnumExtensionLite; @@ -197,6 +198,8 @@ public final class TestUtilLite { builder.setOptionalImportMessage(ImportMessageLite.newBuilder().setD(120).build()); builder.setOptionalPublicImportMessage(PublicImportMessageLite.newBuilder().setE(126).build()); builder.setOptionalLazyMessage(TestAllTypesLite.NestedMessage.newBuilder().setBb(127).build()); + builder.setOptionalUnverifiedLazyMessage( + TestAllTypesLite.NestedMessage.newBuilder().setBb(128).build()); builder.setOptionalNestedEnum(TestAllTypesLite.NestedEnum.BAZ); builder.setOptionalForeignEnum(ForeignEnumLite.FOREIGN_LITE_BAZ); @@ -355,6 +358,9 @@ public final class TestUtilLite { message.setExtension( optionalLazyMessageExtensionLite, TestAllTypesLite.NestedMessage.newBuilder().setBb(127).build()); + message.setExtension( + optionalUnverifiedLazyMessageExtensionLite, + TestAllTypesLite.NestedMessage.newBuilder().setBb(128).build()); message.setExtension(optionalNestedEnumExtensionLite, TestAllTypesLite.NestedEnum.BAZ); message.setExtension(optionalForeignEnumExtensionLite, ForeignEnumLite.FOREIGN_LITE_BAZ); diff --git a/java/core/src/test/proto/com/google/protobuf/map_lite_test.proto b/java/core/src/test/proto/com/google/protobuf/map_lite_test.proto index 7324653f81..ee291e9436 100644 --- a/java/core/src/test/proto/com/google/protobuf/map_lite_test.proto +++ b/java/core/src/test/proto/com/google/protobuf/map_lite_test.proto @@ -119,3 +119,8 @@ message ReservedAsMapFieldWithEnumValue { // null is not a 'reserved word' per se but as a literal needs similar care map null = 10; } + +// https://github.com/protocolbuffers/protobuf/issues/9785 +message MapContainer { + map my_map = 1; +} diff --git a/java/core/src/test/proto/com/google/protobuf/map_test.proto b/java/core/src/test/proto/com/google/protobuf/map_test.proto index 240600f2de..872f4ec9e8 100644 --- a/java/core/src/test/proto/com/google/protobuf/map_test.proto +++ b/java/core/src/test/proto/com/google/protobuf/map_test.proto @@ -118,3 +118,8 @@ message ReservedAsMapFieldWithEnumValue { // null is not a 'reserved word' per se but as a literal needs similar care map null = 10; } + +// https://github.com/protocolbuffers/protobuf/issues/9785 +message MapContainer { + map my_map = 1; +} diff --git a/java/kotlin-lite/src/test/kotlin/com/google/protobuf/Proto2LiteTest.kt b/java/kotlin-lite/src/test/kotlin/com/google/protobuf/Proto2LiteTest.kt index 1f45f654a7..c343ccd236 100644 --- a/java/kotlin-lite/src/test/kotlin/com/google/protobuf/Proto2LiteTest.kt +++ b/java/kotlin-lite/src/test/kotlin/com/google/protobuf/Proto2LiteTest.kt @@ -93,6 +93,7 @@ class Proto2LiteTest { optionalImportMessage = ImportMessageLite.newBuilder().setD(120).build() optionalPublicImportMessage = PublicImportMessageLite.newBuilder().setE(126).build() optionalLazyMessage = nestedMessage { bb = 127 } + optionalUnverifiedLazyMessage = nestedMessage { bb = 128 } optionalNestedEnum = NestedEnum.BAZ optionalForeignEnum = ForeignEnumLite.FOREIGN_LITE_BAZ optionalImportEnum = ImportEnumLite.IMPORT_LITE_BAZ @@ -423,6 +424,8 @@ class Proto2LiteTest { PublicImportMessageLite.newBuilder().setE(126).build() this[UnittestLite.optionalLazyMessageExtensionLite] = TestAllTypesLiteKt.nestedMessage { bb = 127 } + this[UnittestLite.optionalUnverifiedLazyMessageExtensionLite] = + TestAllTypesLiteKt.nestedMessage { bb = 128 } this[UnittestLite.optionalNestedEnumExtensionLite] = NestedEnum.BAZ this[UnittestLite.optionalForeignEnumExtensionLite] = ForeignEnumLite.FOREIGN_LITE_BAZ this[UnittestLite.optionalImportEnumExtensionLite] = ImportEnumLite.IMPORT_LITE_BAZ diff --git a/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt b/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt index 4463bc1492..af797ccb84 100644 --- a/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt +++ b/java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt @@ -98,6 +98,7 @@ class Proto2Test { optionalImportMessage = ImportMessage.newBuilder().setD(120).build() optionalPublicImportMessage = PublicImportMessage.newBuilder().setE(126).build() optionalLazyMessage = nestedMessage { bb = 127 } + optionalUnverifiedLazyMessage = nestedMessage { bb = 128 } optionalNestedEnum = NestedEnum.BAZ optionalForeignEnum = ForeignEnum.FOREIGN_BAZ optionalImportEnum = ImportEnum.IMPORT_BAZ @@ -415,6 +416,8 @@ class Proto2Test { PublicImportMessage.newBuilder().setE(126).build() this[UnittestProto.optionalLazyMessageExtension] = TestAllTypesKt.nestedMessage { bb = 127 } + this[UnittestProto.optionalUnverifiedLazyMessageExtension] = + TestAllTypesKt.nestedMessage { bb = 128 } this[UnittestProto.optionalNestedEnumExtension] = NestedEnum.BAZ this[UnittestProto.optionalForeignEnumExtension] = ForeignEnum.FOREIGN_BAZ this[UnittestProto.optionalImportEnumExtension] = ImportEnum.IMPORT_BAZ diff --git a/python/google/protobuf/descriptor.py b/python/google/protobuf/descriptor.py index b91a8151fc..f5a0caa6bd 100644 --- a/python/google/protobuf/descriptor.py +++ b/python/google/protobuf/descriptor.py @@ -40,13 +40,14 @@ import warnings from google.protobuf.internal import api_implementation _USE_C_DESCRIPTORS = False -if api_implementation.Type() == 'cpp': +if api_implementation.Type() != 'python': # Used by MakeDescriptor in cpp mode import binascii import os - if api_implementation._Version() == 3: - from google3.third_party.upb.python import _message - else: + # pylint: disable=protected-access + _message = api_implementation._c_module + # TODO(jieluo): Remove this import after fix api_implementation + if _message is None: from google.protobuf.pyext import _message _USE_C_DESCRIPTORS = True @@ -601,13 +602,13 @@ class FieldDescriptor(DescriptorBase): self.is_extension = is_extension self.extension_scope = extension_scope self.containing_oneof = containing_oneof - if api_implementation.Type() == 'cpp': + if api_implementation.Type() == 'python': + self._cdescriptor = None + else: if is_extension: self._cdescriptor = _message.default_pool.FindExtensionByName(full_name) else: self._cdescriptor = _message.default_pool.FindFieldByName(full_name) - else: - self._cdescriptor = None @property def camelcase_name(self): @@ -1138,7 +1139,7 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True, Returns: A Descriptor for protobuf messages. """ - if api_implementation.Type() == 'cpp' and build_file_if_cpp: + if api_implementation.Type() != 'python' and build_file_if_cpp: # The C++ implementation requires all descriptors to be backed by the same # definition in the C++ descriptor pool. To do this, we build a # FileDescriptorProto with the same definition as this descriptor and build diff --git a/python/google/protobuf/internal/api_implementation.py b/python/google/protobuf/internal/api_implementation.py index c3f8e65794..a409a80eee 100644 --- a/python/google/protobuf/internal/api_implementation.py +++ b/python/google/protobuf/internal/api_implementation.py @@ -50,24 +50,61 @@ if _api_version == 1: -_default_implementation_type = ('cpp' if _api_version > 0 else 'python') + +def _ApiVersionToImplementationType(api_version): + if api_version == 3: + return 'upb' + if api_version == 2: + return 'cpp' + return 'python' + +# TODO(jieluo): Remove _api_version and only keep implementation_type +# http://b/228103078 +_default_implementation_type = _ApiVersionToImplementationType(_api_version) # This environment variable can be used to switch to a certain implementation # of the Python API, overriding the compile-time constants in the -# _api_implementation module. Right now only 'python' and 'cpp' are valid -# values. Any other value will be ignored. +# _api_implementation module. Right now only 'python', 'cpp' and 'upb' are +# valid values. Any other value will raise error. _implementation_type = os.getenv('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION', _default_implementation_type) -if _implementation_type != 'python': - _implementation_type = 'cpp' +if _implementation_type not in ('python', 'cpp', 'upb'): + raise ValueError('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION {0} is not ' + 'supported. Please set to \'python\', \'cpp\' or ' + '\'upb\'.'.format(_implementation_type)) if 'PyPy' in sys.version and _implementation_type == 'cpp': warnings.warn('PyPy does not work yet with cpp protocol buffers. ' 'Falling back to the python implementation.') _implementation_type = 'python' +_c_module = None + +if _implementation_type == 'cpp': + try: + # pylint: disable=g-import-not-at-top + from google.protobuf.pyext import _message + _c_module = _message + del _message + except ImportError: + # TODO(jieluo): fail back to python + warnings.warn( + 'Selected implementation cpp is not available.') + pass + +if _implementation_type == 'upb': + try: + # pylint: disable=g-import-not-at-top + from google.protobuf.pyext import _upb_message as _message + _c_module = _message + del _message + except ImportError: + warnings.warn('Selected implementation upb is not available. ' + 'Falling back to the python implementation.') + _implementation_type = 'python' + pass # Detect if serialization should be deterministic by default try: @@ -104,12 +141,8 @@ def _SetType(implementation_type): _implementation_type = implementation_type -def _Version(): - """Never use! For protobuf internal use only.""" - return _api_version - - # See comment on 'Type' above. +# TODO(jieluo): Remove the API, it returns a constant. b/228102101 def Version(): return 2 diff --git a/python/google/protobuf/internal/descriptor_test.py b/python/google/protobuf/internal/descriptor_test.py index 1d8286af77..6a8532c408 100644 --- a/python/google/protobuf/internal/descriptor_test.py +++ b/python/google/protobuf/internal/descriptor_test.py @@ -144,7 +144,7 @@ class DescriptorTest(unittest.TestCase): self.assertEqual(self.my_service, self.my_method.containing_service) @unittest.skipIf( - api_implementation.Type() != 'cpp', + api_implementation.Type() == 'python', 'GetDebugString is only available with the cpp implementation', ) def testGetDebugString(self): @@ -457,7 +457,7 @@ class DescriptorTest(unittest.TestCase): self.assertEqual(unittest_pb2.DESCRIPTOR.pool, descriptor_pool.Default()) @unittest.skipIf( - api_implementation.Type() != 'cpp' or api_implementation.Version() != 2, + api_implementation.Type() == 'python', 'Immutability of descriptors is only enforced in v2 implementation') def testImmutableCppDescriptor(self): file_descriptor = unittest_pb2.DESCRIPTOR diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py index e2782a16b7..4002685b02 100644 --- a/python/google/protobuf/internal/message_test.py +++ b/python/google/protobuf/internal/message_test.py @@ -130,7 +130,7 @@ class MessageTest(unittest.TestCase): # b/27494216 end_tag = encoder.TagBytes(1, 4) if (api_implementation.Type() == 'python' or - api_implementation._Version() == 3): + api_implementation.Type() == 'upb'): with self.assertRaises(message.DecodeError) as context: msg.FromString(end_tag) if api_implementation.Type() == 'python': diff --git a/python/google/protobuf/internal/reflection_test.py b/python/google/protobuf/internal/reflection_test.py index a2872a2133..62957d3fde 100644 --- a/python/google/protobuf/internal/reflection_test.py +++ b/python/google/protobuf/internal/reflection_test.py @@ -2020,7 +2020,7 @@ class Proto2ReflectionTest(unittest.TestCase): self.assertRaises(TypeError, proto.IsInitialized, 1, 2, 3) @unittest.skipIf( - api_implementation.Type() != 'cpp' or api_implementation.Version() != 2, + api_implementation.Type() == 'python', 'Errors are only available from the most recent C++ implementation.') def testFileDescriptorErrors(self): file_name = 'test_file_descriptor_errors.proto' @@ -3225,7 +3225,7 @@ class OptionsTest(unittest.TestCase): class ClassAPITest(unittest.TestCase): @unittest.skipIf( - api_implementation.Type() == 'cpp' and api_implementation.Version() == 2, + api_implementation.Type() != 'python', 'C++ implementation requires a call to MakeDescriptor()') @testing_refleaks.SkipReferenceLeakChecker('MakeClass is not repeatable') def testMakeClassWithNestedDescriptor(self): diff --git a/python/google/protobuf/internal/test_util.py b/python/google/protobuf/internal/test_util.py index a6e34ef56d..85284ce528 100644 --- a/python/google/protobuf/internal/test_util.py +++ b/python/google/protobuf/internal/test_util.py @@ -218,6 +218,7 @@ def SetAllNonLazyFields(message): def SetAllFields(message): SetAllNonLazyFields(message) message.optional_lazy_message.bb = 127 + message.optional_unverified_lazy_message.bb = 128 def SetAllExtensions(message): @@ -257,6 +258,7 @@ def SetAllExtensions(message): extensions[pb2.optional_import_message_extension].d = 120 extensions[pb2.optional_public_import_message_extension].e = 126 extensions[pb2.optional_lazy_message_extension].bb = 127 + extensions[pb2.optional_unverified_lazy_message_extension].bb = 128 extensions[pb2.optional_nested_enum_extension] = pb2.TestAllTypes.BAZ extensions[pb2.optional_nested_enum_extension] = pb2.TestAllTypes.BAZ @@ -465,6 +467,7 @@ def ExpectAllFieldsSet(test_case, message): test_case.assertEqual(120, message.optional_import_message.d) test_case.assertEqual(126, message.optional_public_import_message.e) test_case.assertEqual(127, message.optional_lazy_message.bb) + test_case.assertEqual(128, message.optional_unverified_lazy_message.bb) test_case.assertEqual(unittest_pb2.TestAllTypes.BAZ, message.optional_nested_enum) diff --git a/python/google/protobuf/internal/testing_refleaks.py b/python/google/protobuf/internal/testing_refleaks.py index abe88ed578..445406a967 100644 --- a/python/google/protobuf/internal/testing_refleaks.py +++ b/python/google/protobuf/internal/testing_refleaks.py @@ -78,12 +78,23 @@ class ReferenceLeakCheckerMixin(object): oldrefcount = 0 local_result = LocalTestResult(result) + num_flakes = 0 refcount_deltas = [] - for _ in range(self.NB_RUNS): + while len(refcount_deltas) < self.NB_RUNS: oldrefcount = self._getRefcounts() super(ReferenceLeakCheckerMixin, self).run(result=local_result) newrefcount = self._getRefcounts() + # If the GC was able to collect some objects after the call to run() that + # it could not collect before the call, then the counts won't match. + if newrefcount < oldrefcount and num_flakes < 2: + # This result is (probably) a flake -- garbage collectors aren't very + # predictable, but a lower ending refcount is the opposite of the + # failure we are testing for. If the result is repeatable, then we will + # eventually report it, but not after trying to eliminate it. + num_flakes += 1 + continue + num_flakes = 0 refcount_deltas.append(newrefcount - oldrefcount) print(refcount_deltas, self) diff --git a/python/google/protobuf/internal/text_format_test.py b/python/google/protobuf/internal/text_format_test.py index 9b95ce6aad..d1e2748220 100644 --- a/python/google/protobuf/internal/text_format_test.py +++ b/python/google/protobuf/internal/text_format_test.py @@ -584,7 +584,7 @@ class TextFormatMessageToStringTests(TextFormatBase): self.assertEqual(message_proto, parsed_proto) @unittest.skipIf( - api_implementation._Version() == 3, + api_implementation.Type() == 'upb', "upb API doesn't support old UnknownField API. The TextFormat library " "needs to convert to the new API.") def testPrintUnknownFieldsEmbeddedMessageInBytes(self, message_module): diff --git a/python/google/protobuf/internal/unknown_fields_test.py b/python/google/protobuf/internal/unknown_fields_test.py index b639088a3b..a6acc4d329 100644 --- a/python/google/protobuf/internal/unknown_fields_test.py +++ b/python/google/protobuf/internal/unknown_fields_test.py @@ -268,7 +268,7 @@ class UnknownFieldsAccessorsTest(unittest.TestCase): self.InternalCheckUnknownField('optionalgroup', self.all_fields.optionalgroup) - self.assertEqual(97, len(unknown_field_set)) + self.assertEqual(98, len(unknown_field_set)) def testCopyFrom(self): message = unittest_pb2.TestEmptyMessage() @@ -306,7 +306,7 @@ class UnknownFieldsAccessorsTest(unittest.TestCase): self.empty_message.Clear() # All cleared, even unknown fields. self.assertEqual(self.empty_message.SerializeToString(), b'') - self.assertEqual(len(unknown_field_set), 97) + self.assertEqual(len(unknown_field_set), 98) @unittest.skipIf((sys.version_info.major, sys.version_info.minor) < (3, 4), 'tracemalloc requires python 3.4+') @@ -365,7 +365,7 @@ class UnknownFieldsAccessorsTest(unittest.TestCase): def testUnknownExtensions(self): message = unittest_pb2.TestEmptyMessageWithExtensions() message.ParseFromString(self.all_fields_data) - self.assertEqual(len(unknown_fields.UnknownFieldSet(message)), 97) + self.assertEqual(len(message.UnknownFields()), 98) self.assertEqual(message.SerializeToString(), self.all_fields_data) diff --git a/python/google/protobuf/internal/well_known_types_test.py b/python/google/protobuf/internal/well_known_types_test.py index 3618fff26a..a32459a9e6 100644 --- a/python/google/protobuf/internal/well_known_types_test.py +++ b/python/google/protobuf/internal/well_known_types_test.py @@ -426,7 +426,7 @@ class FieldMaskTest(unittest.TestCase): mask = field_mask_pb2.FieldMask() msg_descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR mask.AllFieldsFromDescriptor(msg_descriptor) - self.assertEqual(75, len(mask.paths)) + self.assertEqual(76, len(mask.paths)) self.assertTrue(mask.IsValidForDescriptor(msg_descriptor)) for field in msg_descriptor.fields: self.assertTrue(field.name in mask.paths) diff --git a/python/google/protobuf/message_factory.py b/python/google/protobuf/message_factory.py index 5539410b78..8d65204581 100644 --- a/python/google/protobuf/message_factory.py +++ b/python/google/protobuf/message_factory.py @@ -43,10 +43,10 @@ from google.protobuf.internal import api_implementation from google.protobuf import descriptor_pool from google.protobuf import message -if api_implementation.Type() == 'cpp': - from google.protobuf.pyext import cpp_message as message_impl -else: +if api_implementation.Type() == 'python': from google.protobuf.internal import python_message as message_impl +else: + from google.protobuf.pyext import cpp_message as message_impl # pylint: disable=g-import-not-at-top # The type of all Message classes. diff --git a/python/google/protobuf/pyext/cpp_message.py b/python/google/protobuf/pyext/cpp_message.py index f72b0ce078..ca290299f1 100644 --- a/python/google/protobuf/pyext/cpp_message.py +++ b/python/google/protobuf/pyext/cpp_message.py @@ -38,9 +38,11 @@ __author__ = 'tibell@google.com (Johan Tibell)' from google.protobuf.internal import api_implementation -if api_implementation._Version() == 3: - from google3.third_party.upb.python import _message -else: + +# pylint: disable=protected-access +_message = api_implementation._c_module +# TODO(jieluo): Remove this import after fix api_implementation +if _message is None: from google.protobuf.pyext import _message diff --git a/python/google/protobuf/unknown_fields.py b/python/google/protobuf/unknown_fields.py index 8ae3b7ea2e..d780d3e3a9 100644 --- a/python/google/protobuf/unknown_fields.py +++ b/python/google/protobuf/unknown_fields.py @@ -40,7 +40,7 @@ Simple usage example: from google.protobuf.internal import api_implementation -if api_implementation.Type() == 'cpp': +if api_implementation.Type() != 'python': from google.protobuf.pyext import _message # pylint: disable=g-import-not-at-top else: from google.protobuf.internal import decoder # pylint: disable=g-import-not-at-top diff --git a/src/Makefile.am b/src/Makefile.am index f3e3f272ce..4e9742cf10 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -102,6 +102,7 @@ nobase_include_HEADERS = \ google/protobuf/duration.pb.h \ google/protobuf/dynamic_message.h \ google/protobuf/empty.pb.h \ + google/protobuf/endian.h \ google/protobuf/explicitly_constructed.h \ google/protobuf/extension_set.h \ google/protobuf/extension_set_inl.h \ @@ -142,6 +143,7 @@ nobase_include_HEADERS = \ google/protobuf/port_def.inc \ google/protobuf/port_undef.inc \ google/protobuf/reflection.h \ + google/protobuf/reflection_internal.h \ google/protobuf/reflection_ops.h \ google/protobuf/repeated_field.h \ google/protobuf/repeated_ptr_field.h \ @@ -259,7 +261,6 @@ libprotobuf_la_SOURCES = \ google/protobuf/io/tokenizer.cc \ google/protobuf/map_field.cc \ google/protobuf/message.cc \ - google/protobuf/reflection_internal.h \ google/protobuf/reflection_ops.cc \ google/protobuf/service.cc \ google/protobuf/source_context.pb.cc \ diff --git a/src/google/protobuf/any.pb.h b/src/google/protobuf/any.pb.h index 7dd5d0f82a..8fa3ab074d 100644 --- a/src/google/protobuf/any.pb.h +++ b/src/google/protobuf/any.pb.h @@ -109,10 +109,12 @@ class PROTOBUF_EXPORT Any final : // implements Any ----------------------------------------------- bool PackFrom(const ::PROTOBUF_NAMESPACE_ID::Message& message) { + GOOGLE_DCHECK_NE(&message, this); return _impl_._any_metadata_.PackFrom(GetArena(), message); } bool PackFrom(const ::PROTOBUF_NAMESPACE_ID::Message& message, ::PROTOBUF_NAMESPACE_ID::ConstStringParam type_url_prefix) { + GOOGLE_DCHECK_NE(&message, this); return _impl_._any_metadata_.PackFrom(GetArena(), message, type_url_prefix); } bool UnpackTo(::PROTOBUF_NAMESPACE_ID::Message* message) const { diff --git a/src/google/protobuf/any_test.cc b/src/google/protobuf/any_test.cc index 1d136aa557..a82afb21cb 100644 --- a/src/google/protobuf/any_test.cc +++ b/src/google/protobuf/any_test.cc @@ -176,6 +176,16 @@ TEST(AnyTest, MoveAssignment) { EXPECT_EQ(12345, payload.int32_value()); } +#ifdef PROTOBUF_HAS_DEATH_TEST +#ifndef NDEBUG +TEST(AnyTest, PackSelfDeath) { + google::protobuf::Any any; + EXPECT_DEATH(any.PackFrom(any), "&message"); + EXPECT_DEATH(any.PackFrom(any, ""), "&message"); +} +#endif // !NDEBUG +#endif // PROTOBUF_HAS_DEATH_TEST + } // namespace } // namespace protobuf diff --git a/src/google/protobuf/arenaz_sampler.cc b/src/google/protobuf/arenaz_sampler.cc index 225aeb83a3..0eac693d98 100644 --- a/src/google/protobuf/arenaz_sampler.cc +++ b/src/google/protobuf/arenaz_sampler.cc @@ -55,13 +55,13 @@ void UnsampleSlow(ThreadSafeArenaStats* info) { namespace { PROTOBUF_CONSTINIT std::atomic g_arenaz_enabled{true}; -PROTOBUF_CONSTINIT std::atomic g_arenaz_sample_parameter{1 << 15}; +PROTOBUF_CONSTINIT std::atomic g_arenaz_sample_parameter{1 << 10}; PROTOBUF_THREAD_LOCAL absl::profiling_internal::ExponentialBiased g_exponential_biased_generator; } // namespace -PROTOBUF_THREAD_LOCAL int64_t global_next_sample = 1LL << 15; +PROTOBUF_THREAD_LOCAL int64_t global_next_sample = 1LL << 10; ThreadSafeArenaStats::ThreadSafeArenaStats() { PrepareForSampling(); } ThreadSafeArenaStats::~ThreadSafeArenaStats() = default; diff --git a/src/google/protobuf/compiler/cpp/generator.cc b/src/google/protobuf/compiler/cpp/generator.cc index 8d2cfecdfc..63a2bcebed 100644 --- a/src/google/protobuf/compiler/cpp/generator.cc +++ b/src/google/protobuf/compiler/cpp/generator.cc @@ -135,12 +135,10 @@ bool CppGenerator::Generate(const FileDescriptor* file, .insert(options[i].second.substr(pos, next_pos - pos)); pos = next_pos + 1; } while (pos < options[i].second.size()); - } else if (options[i].first == "verified_lazy_message_sets") { - file_options.unverified_lazy_message_sets = false; + } else if (options[i].first == "verified_lazy") { + file_options.unverified_lazy = false; } else if (options[i].first == "unverified_lazy_message_sets") { file_options.unverified_lazy_message_sets = true; - } else if (options[i].first == "eagerly_verified_lazy") { - file_options.eagerly_verified_lazy = true; } else if (options[i].first == "message_owned_arena_trial") { file_options.message_owned_arena_trial = true; } else if (options[i].first == "force_eagerly_verified_lazy") { diff --git a/src/google/protobuf/compiler/cpp/helpers.cc b/src/google/protobuf/compiler/cpp/helpers.cc index f959a42189..685613f6ad 100644 --- a/src/google/protobuf/compiler/cpp/helpers.cc +++ b/src/google/protobuf/compiler/cpp/helpers.cc @@ -178,11 +178,6 @@ void SetIntVar(const Options& options, const std::string& type, std::map* variables) { (*variables)[type] = IntTypeName(options, type); } -bool IsEagerlyVerifiedLazyImpl(const FieldDescriptor* field, - const Options& options, - MessageSCCAnalyzer* scc_analyzer) { - return false; -} // Returns true if the message can potentially allocate memory for its field. // This is used to determine if message-owned arena will be useful. @@ -195,7 +190,23 @@ bool AllocExpected(const Descriptor* descriptor) { bool IsLazy(const FieldDescriptor* field, const Options& options, MessageSCCAnalyzer* scc_analyzer) { return IsLazilyVerifiedLazy(field, options) || - IsEagerlyVerifiedLazyImpl(field, options, scc_analyzer); + IsEagerlyVerifiedLazy(field, options, scc_analyzer); +} + +bool IsEagerlyVerifiedLazyByProfile(const FieldDescriptor* field, + const Options& options, + MessageSCCAnalyzer* scc_analyzer) { + return false; +} + +bool IsEagerlyVerifiedLazy(const FieldDescriptor* field, const Options& options, + MessageSCCAnalyzer* scc_analyzer) { + return false; +} + +bool IsLazilyVerifiedLazy(const FieldDescriptor* field, + const Options& options) { + return false; } void SetCommonVars(const Options& options, diff --git a/src/google/protobuf/compiler/cpp/helpers.h b/src/google/protobuf/compiler/cpp/helpers.h index f51e55f290..fece19366b 100644 --- a/src/google/protobuf/compiler/cpp/helpers.h +++ b/src/google/protobuf/compiler/cpp/helpers.h @@ -364,21 +364,16 @@ inline bool IsExplicitLazy(const FieldDescriptor* field) { return field->options().lazy() || field->options().unverified_lazy(); } -inline bool IsLazilyVerifiedLazy(const FieldDescriptor* field, - const Options& options) { - // TODO(b/211906113): Make lazy() imply eagerly verified lazy. - return IsExplicitLazy(field) && !field->is_repeated() && - field->type() == FieldDescriptor::TYPE_MESSAGE && - GetOptimizeFor(field->file(), options) != FileOptions::LITE_RUNTIME && - !options.opensource_runtime; -} +// Returns true if "field" is a message field that is backed by LazyField per +// profile (go/pdlazy). +bool IsEagerlyVerifiedLazyByProfile(const FieldDescriptor* field, + const Options& options, + MessageSCCAnalyzer* scc_analyzer); -inline bool IsEagerlyVerifiedLazy(const FieldDescriptor* field, - const Options& options, - MessageSCCAnalyzer* scc_analyzer) { - // TODO(b/211906113): Make lazy() imply eagerly verified lazy. - return IsLazy(field, options, scc_analyzer) && !IsExplicitLazy(field); -} +bool IsEagerlyVerifiedLazy(const FieldDescriptor* field, const Options& options, + MessageSCCAnalyzer* scc_analyzer); + +bool IsLazilyVerifiedLazy(const FieldDescriptor* field, const Options& options); inline bool IsFieldUsed(const FieldDescriptor* /* field */, const Options& /* options */) { diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index 87f7612a76..1990c38e84 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -1511,11 +1511,13 @@ void MessageGenerator::GenerateClassDefinition(io::Printer* printer) { if (HasDescriptorMethods(descriptor_->file(), options_)) { format( "bool PackFrom(const ::$proto_ns$::Message& message) {\n" + " $DCHK$_NE(&message, this);\n" " return $any_metadata$.PackFrom(GetArena(), message);\n" "}\n" "bool PackFrom(const ::$proto_ns$::Message& message,\n" " ::PROTOBUF_NAMESPACE_ID::ConstStringParam " "type_url_prefix) {\n" + " $DCHK$_NE(&message, this);\n" " return $any_metadata$.PackFrom(GetArena(), message, " "type_url_prefix);\n" "}\n" @@ -2248,7 +2250,7 @@ std::pair MessageGenerator::GenerateOffsets( // // Embed whether the field is eagerly verified lazy or inlined string to the // LSB of the offset. - if (IsEagerlyVerifiedLazy(field, options_, scc_analyzer_)) { + if (IsEagerlyVerifiedLazyByProfile(field, options_, scc_analyzer_)) { format(" | 0x1u // eagerly verified lazy\n"); } else if (IsStringInlined(field, options_)) { format(" | 0x1u // inlined\n"); diff --git a/src/google/protobuf/compiler/cpp/message_size_unittest.cc b/src/google/protobuf/compiler/cpp/message_size_unittest.cc index 12093509a9..761988bf66 100644 --- a/src/google/protobuf/compiler/cpp/message_size_unittest.cc +++ b/src/google/protobuf/compiler/cpp/message_size_unittest.cc @@ -29,9 +29,9 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include -#include #include #include +#include namespace google { namespace protobuf { diff --git a/src/google/protobuf/compiler/cpp/options.h b/src/google/protobuf/compiler/cpp/options.h index d7fe6a28d2..148866ff6e 100644 --- a/src/google/protobuf/compiler/cpp/options.h +++ b/src/google/protobuf/compiler/cpp/options.h @@ -80,7 +80,7 @@ struct Options { bool annotate_accessor = false; bool unused_field_stripping = false; bool unverified_lazy_message_sets = false; - bool eagerly_verified_lazy = true; + bool unverified_lazy = true; bool profile_driven_inline_string = true; bool message_owned_arena_trial = false; bool force_split = false; diff --git a/src/google/protobuf/compiler/parser_unittest.cc b/src/google/protobuf/compiler/parser_unittest.cc index 9027891ec1..2d681d957f 100644 --- a/src/google/protobuf/compiler/parser_unittest.cc +++ b/src/google/protobuf/compiler/parser_unittest.cc @@ -1861,7 +1861,8 @@ TEST_F(ParserValidationErrorTest, FieldNumberError) { "message Foo {\n" " optional int32 bar = 0;\n" "}\n", - "1:23: Field numbers must be positive integers.\n"); + "1:23: Field numbers must be positive integers.\n" + "1:23: Suggested field numbers for Foo: 1\n"); } TEST_F(ParserValidationErrorTest, FieldExtendeeError) { @@ -1926,7 +1927,8 @@ TEST_F(ParserValidationErrorTest, ExtensionRangeNumberError) { "message Foo {\n" " extensions 0;\n" "}\n", - "1:13: Extension numbers must be positive integers.\n"); + "1:13: Extension numbers must be positive integers.\n" + "1:13: Suggested field numbers for Foo: 1\n"); } TEST_F(ParserValidationErrorTest, Proto3ExtensionError) { diff --git a/src/google/protobuf/compiler/python/generator.cc b/src/google/protobuf/compiler/python/generator.cc index 7c6a412e36..d8d6d74923 100644 --- a/src/google/protobuf/compiler/python/generator.cc +++ b/src/google/protobuf/compiler/python/generator.cc @@ -90,6 +90,7 @@ std::string ModuleAlias(const std::string& filename) { // in proto2/public/reflection.py. const char kDescriptorKey[] = "DESCRIPTOR"; + // file output by this generator. void PrintTopBoilerplate(io::Printer* printer, const FileDescriptor* file, bool descriptor_proto) { diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 205cf86d2d..97160fdeb7 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -3699,6 +3700,28 @@ class DescriptorBuilder { FileDescriptorTables* file_tables_; std::set dependencies_; + struct MessageHints { + int fields_to_suggest = 0; + const Message* first_reason = nullptr; + DescriptorPool::ErrorCollector::ErrorLocation first_reason_location = + DescriptorPool::ErrorCollector::ErrorLocation::OTHER; + + void RequestHintOnFieldNumbers( + const Message& reason, + DescriptorPool::ErrorCollector::ErrorLocation reason_location, + int fields_count = 1) { + constexpr int kMaxSuggestions = 3; + fields_to_suggest = + std::min(kMaxSuggestions, + fields_to_suggest + std::min(kMaxSuggestions, fields_count)); + if (first_reason) return; + first_reason = &reason; + first_reason_location = reason_location; + } + }; + + std::unordered_map message_hints_; + // unused_dependency_ is used to record the unused imported files. // Note: public import is not considered. std::set unused_dependency_; @@ -3912,6 +3935,8 @@ class DescriptorBuilder { const ServiceDescriptorProto& proto); void CrossLinkMethod(MethodDescriptor* method, const MethodDescriptorProto& proto); + void SuggestFieldNumbers(FileDescriptor* file, + const FileDescriptorProto& proto); // Must be run only after cross-linking. void InterpretOptions(); @@ -5267,6 +5292,10 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( // Cross-link. CrossLinkFile(result, proto); + if (!message_hints_.empty()) { + SuggestFieldNumbers(result, proto); + } + // Interpret any remaining uninterpreted options gathered into // options_to_interpret_ during descriptor building. Cross-linking has made // extension options known, so all interpretations should now succeed. @@ -5441,6 +5470,8 @@ void DescriptorBuilder::BuildMessage(const DescriptorProto& proto, for (int j = 0; j < result->extension_range_count(); j++) { const Descriptor::ExtensionRange* range = result->extension_range(j); if (range->start <= field->number() && field->number() < range->end) { + message_hints_[result].RequestHintOnFieldNumbers( + proto.extension_range(j), DescriptorPool::ErrorCollector::NUMBER); AddError( field->full_name(), proto.extension_range(j), DescriptorPool::ErrorCollector::NUMBER, @@ -5452,6 +5483,8 @@ void DescriptorBuilder::BuildMessage(const DescriptorProto& proto, for (int j = 0; j < result->reserved_range_count(); j++) { const Descriptor::ReservedRange* range = result->reserved_range(j); if (range->start <= field->number() && field->number() < range->end) { + message_hints_[result].RequestHintOnFieldNumbers( + proto.reserved_range(j), DescriptorPool::ErrorCollector::NUMBER); AddError(field->full_name(), proto.reserved_range(j), DescriptorPool::ErrorCollector::NUMBER, strings::Substitute("Field \"$0\" uses reserved number $1.", @@ -5698,6 +5731,8 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, } if (result->number() <= 0) { + message_hints_[parent].RequestHintOnFieldNumbers( + proto, DescriptorPool::ErrorCollector::NUMBER); AddError(result->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, "Field numbers must be positive integers."); } else if (!is_extension && result->number() > FieldDescriptor::kMaxNumber) { @@ -5709,11 +5744,15 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, // This avoids cross-linking issues that arise when attempting to check if // the extendee is a message_set_wire_format message, which has a higher max // on extension numbers. + message_hints_[parent].RequestHintOnFieldNumbers( + proto, DescriptorPool::ErrorCollector::NUMBER); AddError(result->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, strings::Substitute("Field numbers cannot be greater than $0.", FieldDescriptor::kMaxNumber)); } else if (result->number() >= FieldDescriptor::kFirstReservedNumber && result->number() <= FieldDescriptor::kLastReservedNumber) { + message_hints_[parent].RequestHintOnFieldNumbers( + proto, DescriptorPool::ErrorCollector::NUMBER); AddError(result->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, strings::Substitute( "Field numbers $0 through $1 are reserved for the protocol " @@ -5778,6 +5817,9 @@ void DescriptorBuilder::BuildExtensionRange( result->start = proto.start(); result->end = proto.end(); if (result->start <= 0) { + message_hints_[parent].RequestHintOnFieldNumbers( + proto, DescriptorPool::ErrorCollector::NUMBER, + std::max(0, result->end - result->start)); AddError(parent->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, "Extension numbers must be positive integers."); } @@ -5815,6 +5857,9 @@ void DescriptorBuilder::BuildReservedRange( result->start = proto.start(); result->end = proto.end(); if (result->start <= 0) { + message_hints_[parent].RequestHintOnFieldNumbers( + proto, DescriptorPool::ErrorCollector::NUMBER, + std::max(0, result->end - result->start)); AddError(parent->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, "Reserved numbers must be positive integers."); } @@ -6649,6 +6694,78 @@ void DescriptorBuilder::CrossLinkMethod(MethodDescriptor* method, method->output_type_.Set(output_type.descriptor()); } } + +void DescriptorBuilder::SuggestFieldNumbers(FileDescriptor* file, + const FileDescriptorProto& proto) { + for (int message_index = 0; message_index < file->message_type_count(); + message_index++) { + const Descriptor* message = &file->message_types_[message_index]; + auto* hints = FindOrNull(message_hints_, message); + if (!hints) continue; + int fields_to_suggest = hints->fields_to_suggest; + if (fields_to_suggest <= 0) continue; + struct Range { + int from; + int to; + }; + std::vector used_ordinals; + auto add_ordinal = [&](int ordinal) { + if (!used_ordinals.empty() && + used_ordinals.back().to < FieldDescriptor::kMaxNumber && + ordinal == used_ordinals.back().to + 1) { + used_ordinals.back().to = ordinal; + } else { + used_ordinals.push_back({ordinal, ordinal}); + } + }; + auto add_range = [&](int from, int to) { + from = std::max(0, std::min(FieldDescriptor::kMaxNumber, from)); + to = std::max(0, std::min(FieldDescriptor::kMaxNumber, to)); + if (from > to) return; + used_ordinals.push_back({from, to}); + }; + for (int i = 0; i < message->field_count(); i++) { + add_ordinal(message->field(i)->number()); + } + for (int i = 0; i < message->extension_count(); i++) { + add_ordinal(message->extension(i)->number()); + } + for (int i = 0; i < message->reserved_range_count(); i++) { + auto range = message->reserved_range(i); + add_range(range->start, range->end - 1); + } + for (int i = 0; i < message->extension_range_count(); i++) { + auto range = message->extension_range(i); + add_range(range->start, range->end - 1); + } + used_ordinals.push_back( + {FieldDescriptor::kMaxNumber, std::numeric_limits::max()}); + used_ordinals.push_back({FieldDescriptor::kFirstReservedNumber, + FieldDescriptor::kLastReservedNumber}); + std::sort(used_ordinals.begin(), used_ordinals.end(), + [](Range lhs, Range rhs) { + return std::tie(lhs.from, lhs.to) < std::tie(rhs.from, rhs.to); + }); + int current_ordinal = 1; + std::stringstream id_list; + id_list << "Suggested field numbers for " << message->full_name() << ": "; + const char* separator = ""; + for (auto& current_range : used_ordinals) { + while (current_ordinal < current_range.from && fields_to_suggest > 0) { + id_list << separator << current_ordinal++; + separator = ", "; + fields_to_suggest--; + } + if (fields_to_suggest == 0) break; + current_ordinal = std::max(current_ordinal, current_range.to + 1); + } + if (hints->first_reason) { + AddError(message->full_name(), *hints->first_reason, + hints->first_reason_location, id_list.str()); + } + } +} + // ------------------------------------------------------------------- #define VALIDATE_OPTIONS_FROM_ARRAY(descriptor, array_name, type) \ diff --git a/src/google/protobuf/descriptor_unittest.cc b/src/google/protobuf/descriptor_unittest.cc index 5f2a6502bf..d8474fce02 100644 --- a/src/google/protobuf/descriptor_unittest.cc +++ b/src/google/protobuf/descriptor_unittest.cc @@ -4122,7 +4122,8 @@ TEST_F(ValidationErrorTest, FieldInExtensionRange) { "foo.proto: Foo.bar: NUMBER: Extension range 10 to 19 includes field " "\"bar\" (10).\n" "foo.proto: Foo.baz: NUMBER: Extension range 10 to 19 includes field " - "\"baz\" (19).\n"); + "\"baz\" (19).\n" + "foo.proto: Foo: NUMBER: Suggested field numbers for Foo: 1, 2\n"); } TEST_F(ValidationErrorTest, OverlappingExtensionRanges) { @@ -4151,7 +4152,8 @@ TEST_F(ValidationErrorTest, ReservedFieldError) { " reserved_range { start: 10 end: 20 }" "}", - "foo.proto: Foo.foo: NUMBER: Field \"foo\" uses reserved number 15.\n"); + "foo.proto: Foo.foo: NUMBER: Field \"foo\" uses reserved number 15.\n" + "foo.proto: Foo: NUMBER: Suggested field numbers for Foo: 1\n"); } TEST_F(ValidationErrorTest, ReservedExtensionRangeError) { @@ -4484,7 +4486,8 @@ TEST_F(ValidationErrorTest, NegativeFieldNumber) { "}" "}", - "foo.proto: Foo.foo: NUMBER: Field numbers must be positive integers.\n"); + "foo.proto: Foo.foo: NUMBER: Field numbers must be positive integers.\n" + "foo.proto: Foo: NUMBER: Suggested field numbers for Foo: 1\n"); } TEST_F(ValidationErrorTest, HugeFieldNumber) { @@ -4497,7 +4500,8 @@ TEST_F(ValidationErrorTest, HugeFieldNumber) { "}", "foo.proto: Foo.foo: NUMBER: Field numbers cannot be greater than " - "536870911.\n"); + "536870911.\n" + "foo.proto: Foo: NUMBER: Suggested field numbers for Foo: 1\n"); } TEST_F(ValidationErrorTest, ReservedFieldNumber) { @@ -4518,7 +4522,8 @@ TEST_F(ValidationErrorTest, ReservedFieldNumber) { "foo.proto: Foo.bar: NUMBER: Field numbers 19000 through 19999 are " "reserved for the protocol buffer library implementation.\n" "foo.proto: Foo.baz: NUMBER: Field numbers 19000 through 19999 are " - "reserved for the protocol buffer library implementation.\n"); + "reserved for the protocol buffer library implementation.\n" + "foo.proto: Foo: NUMBER: Suggested field numbers for Foo: 1, 2\n"); } TEST_F(ValidationErrorTest, ExtensionMissingExtendee) { @@ -4716,7 +4721,8 @@ TEST_F(ValidationErrorTest, NegativeExtensionRangeNumber) { " extension_range { start: -10 end: -1 }" "}", - "foo.proto: Foo: NUMBER: Extension numbers must be positive integers.\n"); + "foo.proto: Foo: NUMBER: Extension numbers must be positive integers.\n" + "foo.proto: Foo: NUMBER: Suggested field numbers for Foo: 1, 2, 3\n"); } TEST_F(ValidationErrorTest, HugeExtensionRangeNumber) { diff --git a/src/google/protobuf/endian.h b/src/google/protobuf/endian.h new file mode 100644 index 0000000000..e0ee6cdf28 --- /dev/null +++ b/src/google/protobuf/endian.h @@ -0,0 +1,198 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_ENDIAN_H__ +#define GOOGLE_PROTOBUF_ENDIAN_H__ + +#if defined(_MSC_VER) +#include +#endif + +#include + +// Must be included last. +#include + +namespace google { +namespace protobuf { +namespace internal { + +inline uint64_t BSwap64(uint64_t host_int) { +#if defined(PROTOBUF_BUILTIN_BSWAP64) + return PROTOBUF_BUILTIN_BSWAP64(host_int); +#elif defined(_MSC_VER) + return _byteswap_uint64(host_int); +#else + return (((host_int & uint64_t{0xFF}) << 56) | + ((host_int & uint64_t{0xFF00}) << 40) | + ((host_int & uint64_t{0xFF0000}) << 24) | + ((host_int & uint64_t{0xFF000000}) << 8) | + ((host_int & uint64_t{0xFF00000000}) >> 8) | + ((host_int & uint64_t{0xFF0000000000}) >> 24) | + ((host_int & uint64_t{0xFF000000000000}) >> 40) | + ((host_int & uint64_t{0xFF00000000000000}) >> 56)); +#endif +} + +inline uint32_t BSwap32(uint32_t host_int) { +#if defined(PROTOBUF_BUILTIN_BSWAP32) + return PROTOBUF_BUILTIN_BSWAP32(host_int); +#elif defined(_MSC_VER) + return _byteswap_ulong(host_int); +#else + return (((host_int & uint32_t{0xFF}) << 24) | + ((host_int & uint32_t{0xFF00}) << 8) | + ((host_int & uint32_t{0xFF0000}) >> 8) | + ((host_int & uint32_t{0xFF000000}) >> 24)); +#endif +} + +inline uint16_t BSwap16(uint16_t host_int) { +#if defined(PROTOBUF_BUILTIN_BSWAP16) + return PROTOBUF_BUILTIN_BSWAP16(host_int); +#elif defined(_MSC_VER) + return _byteswap_ushort(host_int); +#else + return (((host_int & uint16_t{0xFF}) << 8) | + ((host_int & uint16_t{0xFF00}) >> 8)); +#endif +} + +namespace little_endian { + +inline uint16_t FromHost(uint16_t value) { +#if defined(PROTOBUF_BIG_ENDIAN) + return BSwap16(value); +#else + return value; +#endif +} + +inline uint32_t FromHost(uint32_t value) { +#if defined(PROTOBUF_BIG_ENDIAN) + return BSwap32(value); +#else + return value; +#endif +} + +inline uint64_t FromHost(uint64_t value) { +#if defined(PROTOBUF_BIG_ENDIAN) + return BSwap64(value); +#else + return value; +#endif +} + +inline uint16_t ToHost(uint16_t value) { +#if defined(PROTOBUF_BIG_ENDIAN) + return BSwap16(value); +#else + return value; +#endif +} + +inline uint32_t ToHost(uint32_t value) { +#if defined(PROTOBUF_BIG_ENDIAN) + return BSwap32(value); +#else + return value; +#endif +} + +inline uint64_t ToHost(uint64_t value) { +#if defined(PROTOBUF_BIG_ENDIAN) + return BSwap64(value); +#else + return value; +#endif +} + +} // namespace little_endian + +namespace big_endian { + +inline uint16_t FromHost(uint16_t value) { +#if defined(PROTOBUF_BIG_ENDIAN) + return value; +#else + return BSwap16(value); +#endif +} + +inline uint32_t FromHost(uint32_t value) { +#if defined(PROTOBUF_BIG_ENDIAN) + return value; +#else + return BSwap32(value); +#endif +} + +inline uint64_t FromHost(uint64_t value) { +#if defined(PROTOBUF_BIG_ENDIAN) + return value; +#else + return BSwap64(value); +#endif +} + +inline uint16_t ToHost(uint16_t value) { +#if defined(PROTOBUF_BIG_ENDIAN) + return value; +#else + return BSwap16(value); +#endif +} + +inline uint32_t ToHost(uint32_t value) { +#if defined(PROTOBUF_BIG_ENDIAN) + return value; +#else + return BSwap32(value); +#endif +} + +inline uint64_t ToHost(uint64_t value) { +#if defined(PROTOBUF_BIG_ENDIAN) + return value; +#else + return BSwap64(value); +#endif +} + +} // namespace big_endian + +} // namespace internal +} // namespace protobuf +} // namespace google + +#include + +#endif // GOOGLE_PROTOBUF_ENDIAN_H__ diff --git a/src/google/protobuf/extension_set.cc b/src/google/protobuf/extension_set.cc index a0bccbea5f..fada4f5a98 100644 --- a/src/google/protobuf/extension_set.cc +++ b/src/google/protobuf/extension_set.cc @@ -69,24 +69,6 @@ inline WireFormatLite::CppType cpp_type(FieldType type) { return WireFormatLite::FieldTypeToCppType(real_type(type)); } -inline bool is_packable(WireFormatLite::WireType type) { - switch (type) { - case WireFormatLite::WIRETYPE_VARINT: - case WireFormatLite::WIRETYPE_FIXED64: - case WireFormatLite::WIRETYPE_FIXED32: - return true; - case WireFormatLite::WIRETYPE_LENGTH_DELIMITED: - case WireFormatLite::WIRETYPE_START_GROUP: - case WireFormatLite::WIRETYPE_END_GROUP: - return false; - - // Do not add a default statement. Let the compiler complain when someone - // adds a new wire type. - } - GOOGLE_LOG(FATAL) << "can't reach here."; - return false; -} - // Registry stuff. // Note that we cannot use hetererogeneous lookup for std containers since we @@ -139,8 +121,6 @@ const ExtensionInfo* FindRegisteredExtension(const MessageLite* extendee, } // namespace -ExtensionFinder::~ExtensionFinder() {} - bool GeneratedExtensionFinder::Find(int number, ExtensionInfo* output) { const ExtensionInfo* extension = FindRegisteredExtension(extendee_, number); if (extension == nullptr) { @@ -1247,40 +1227,6 @@ bool ExtensionSet::IsInitialized() const { return true; } -bool ExtensionSet::FindExtensionInfoFromTag(uint32_t tag, - ExtensionFinder* extension_finder, - int* field_number, - ExtensionInfo* extension, - bool* was_packed_on_wire) { - *field_number = WireFormatLite::GetTagFieldNumber(tag); - WireFormatLite::WireType wire_type = WireFormatLite::GetTagWireType(tag); - return FindExtensionInfoFromFieldNumber(wire_type, *field_number, - extension_finder, extension, - was_packed_on_wire); -} - -bool ExtensionSet::FindExtensionInfoFromFieldNumber( - int wire_type, int field_number, ExtensionFinder* extension_finder, - ExtensionInfo* extension, bool* was_packed_on_wire) const { - if (!extension_finder->Find(field_number, extension)) { - return false; - } - - WireFormatLite::WireType expected_wire_type = - WireFormatLite::WireTypeForFieldType(real_type(extension->type)); - - // Check if this is a packed field. - *was_packed_on_wire = false; - if (extension->is_repeated && - wire_type == WireFormatLite::WIRETYPE_LENGTH_DELIMITED && - is_packable(expected_wire_type)) { - *was_packed_on_wire = true; - return true; - } - // Otherwise the wire type must match. - return expected_wire_type == wire_type; -} - const char* ExtensionSet::ParseField(uint64_t tag, const char* ptr, const MessageLite* extendee, internal::InternalMetadata* metadata, diff --git a/src/google/protobuf/extension_set.h b/src/google/protobuf/extension_set.h index 026b102ab7..0e6d052110 100644 --- a/src/google/protobuf/extension_set.h +++ b/src/google/protobuf/extension_set.h @@ -147,26 +147,20 @@ struct ExtensionInfo { LazyEagerVerifyFnType lazy_eager_verify_func = nullptr; }; -// Abstract interface for an object which looks up extension definitions. Used -// when parsing. -class PROTOBUF_EXPORT ExtensionFinder { - public: - virtual ~ExtensionFinder(); - - // Find the extension with the given containing type and number. - virtual bool Find(int number, ExtensionInfo* output) = 0; -}; +// An ExtensionFinder is an object which looks up extension definitions. It +// must implement this method: +// +// bool Find(int number, ExtensionInfo* output); -// Implementation of ExtensionFinder which finds extensions defined in .proto -// files which have been compiled into the binary. -class PROTOBUF_EXPORT GeneratedExtensionFinder : public ExtensionFinder { +// GeneratedExtensionFinder is an ExtensionFinder which finds extensions +// defined in .proto files which have been compiled into the binary. +class PROTOBUF_EXPORT GeneratedExtensionFinder { public: explicit GeneratedExtensionFinder(const MessageLite* extendee) : extendee_(extendee) {} - ~GeneratedExtensionFinder() override {} // Returns true and fills in *output if found, otherwise returns false. - bool Find(int number, ExtensionInfo* output) override; + bool Find(int number, ExtensionInfo* output); private: const MessageLite* extendee_; @@ -746,22 +740,71 @@ class PROTOBUF_EXPORT ExtensionSet { const Extension& other_extension, Arena* other_arena); + inline static bool is_packable(WireFormatLite::WireType type) { + switch (type) { + case WireFormatLite::WIRETYPE_VARINT: + case WireFormatLite::WIRETYPE_FIXED64: + case WireFormatLite::WIRETYPE_FIXED32: + return true; + case WireFormatLite::WIRETYPE_LENGTH_DELIMITED: + case WireFormatLite::WIRETYPE_START_GROUP: + case WireFormatLite::WIRETYPE_END_GROUP: + return false; + + // Do not add a default statement. Let the compiler complain when + // someone + // adds a new wire type. + } + PROTOBUF_ASSUME(false); // switch handles all possible enum values + return false; + } + // Returns true and fills field_number and extension if extension is found. // Note to support packed repeated field compatibility, it also fills whether // the tag on wire is packed, which can be different from // extension->is_packed (whether packed=true is specified). + template bool FindExtensionInfoFromTag(uint32_t tag, ExtensionFinder* extension_finder, int* field_number, ExtensionInfo* extension, - bool* was_packed_on_wire); + bool* was_packed_on_wire) { + *field_number = WireFormatLite::GetTagFieldNumber(tag); + WireFormatLite::WireType wire_type = WireFormatLite::GetTagWireType(tag); + return FindExtensionInfoFromFieldNumber(wire_type, *field_number, + extension_finder, extension, + was_packed_on_wire); + } // Returns true and fills extension if extension is found. // Note to support packed repeated field compatibility, it also fills whether // the tag on wire is packed, which can be different from // extension->is_packed (whether packed=true is specified). + template bool FindExtensionInfoFromFieldNumber(int wire_type, int field_number, ExtensionFinder* extension_finder, ExtensionInfo* extension, - bool* was_packed_on_wire) const; + bool* was_packed_on_wire) const { + if (!extension_finder->Find(field_number, extension)) { + return false; + } + + GOOGLE_DCHECK(extension->type > 0 && + extension->type <= WireFormatLite::MAX_FIELD_TYPE); + auto real_type = static_cast(extension->type); + + WireFormatLite::WireType expected_wire_type = + WireFormatLite::WireTypeForFieldType(real_type); + + // Check if this is a packed field. + *was_packed_on_wire = false; + if (extension->is_repeated && + wire_type == WireFormatLite::WIRETYPE_LENGTH_DELIMITED && + is_packable(expected_wire_type)) { + *was_packed_on_wire = true; + return true; + } + // Otherwise the wire type must match. + return expected_wire_type == wire_type; + } // Find the prototype for a LazyMessage from the extension registry. Returns // null if the extension is not found. diff --git a/src/google/protobuf/extension_set_heavy.cc b/src/google/protobuf/extension_set_heavy.cc index 71c26c56cd..a4bb94860c 100644 --- a/src/google/protobuf/extension_set_heavy.cc +++ b/src/google/protobuf/extension_set_heavy.cc @@ -61,15 +61,14 @@ namespace internal { // Implementation of ExtensionFinder which finds extensions in a given // DescriptorPool, using the given MessageFactory to construct sub-objects. // This class is implemented in extension_set_heavy.cc. -class DescriptorPoolExtensionFinder : public ExtensionFinder { +class DescriptorPoolExtensionFinder { public: DescriptorPoolExtensionFinder(const DescriptorPool* pool, MessageFactory* factory, const Descriptor* containing_type) : pool_(pool), factory_(factory), containing_type_(containing_type) {} - ~DescriptorPoolExtensionFinder() override {} - bool Find(int number, ExtensionInfo* output) override; + bool Find(int number, ExtensionInfo* output); private: const DescriptorPool* pool_; diff --git a/src/google/protobuf/generated_message_tctable_impl.h b/src/google/protobuf/generated_message_tctable_impl.h index c2be089bc4..21fa5332d3 100644 --- a/src/google/protobuf/generated_message_tctable_impl.h +++ b/src/google/protobuf/generated_message_tctable_impl.h @@ -400,6 +400,13 @@ class PROTOBUF_EXPORT TcParser final { return *target; } + template + static inline T ReadAt(const void* x, size_t offset) { + T out; + memcpy(&out, static_cast(x) + offset, sizeof(T)); + return out; + } + // Mini parsing: // // This function parses a field from incoming data based on metadata stored in diff --git a/src/google/protobuf/generated_message_tctable_lite.cc b/src/google/protobuf/generated_message_tctable_lite.cc index ff9f1496f3..9993811dca 100644 --- a/src/google/protobuf/generated_message_tctable_lite.cc +++ b/src/google/protobuf/generated_message_tctable_lite.cc @@ -84,10 +84,10 @@ PROTOBUF_NOINLINE const char* TcParser::ParseLoop( MessageLite* msg, const char* ptr, ParseContext* ctx, const TcParseTableBase* table) { ScopedArenaSwap saved(msg, ctx); - const uint32_t has_bits_offset = table->has_bits_offset; while (!ctx->Done(&ptr)) { - uint64_t hasbits = 0; - if (has_bits_offset) hasbits = RefAt(msg, has_bits_offset); + // Unconditionally read has bits, even if we don't have has bits. + // has_bits_offset will be 0 and we will just read something valid. + uint64_t hasbits = ReadAt(msg, table->has_bits_offset); ptr = TagDispatch(msg, ptr, ctx, table, hasbits, {}); if (ptr == nullptr) break; if (ctx->LastTag() != 1) break; // Ended on terminating tag @@ -301,7 +301,7 @@ StringPiece TcParser::FieldName(const TcParseTableBase* table, const char* TcParser::MiniParse(PROTOBUF_TC_PARAM_DECL) { uint32_t tag; - ptr = ReadTag(ptr, &tag); + ptr = ReadTagInlined(ptr, &tag); if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) return nullptr; auto* entry = FindFieldEntry(table, tag >> 3); @@ -371,13 +371,13 @@ const char* TcParser::SingularParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL) { auto saved_tag = UnalignedLoad(ptr); ptr += sizeof(TagType); hasbits |= (uint64_t{1} << data.hasbit_idx()); + SyncHasbits(msg, hasbits, table); auto& field = RefAt(msg, data.offset()); if (field == nullptr) { const MessageLite* default_instance = table->field_aux(data.aux_idx())->message_default; field = default_instance->New(ctx->data().arena); } - SyncHasbits(msg, hasbits, table); if (group_coding) { return ctx->ParseGroup(field, ptr, FastDecodeTag(saved_tag)); } @@ -456,7 +456,7 @@ PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularFixed( } ptr += sizeof(TagType); // Consume tag hasbits |= (uint64_t{1} << data.hasbit_idx()); - std::memcpy(Offset(msg, data.offset()), ptr, sizeof(LayoutType)); + RefAt(msg, data.offset()) = UnalignedLoad(ptr); ptr += sizeof(LayoutType); PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); } @@ -501,7 +501,7 @@ PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedFixed( auto expected_tag = UnalignedLoad(ptr); do { ptr += sizeof(TagType); - std::memcpy(elem + (idx++), ptr, sizeof(LayoutType)); + elem[idx++] = UnalignedLoad(ptr); ptr += sizeof(LayoutType); if (idx >= space) break; if (!ctx->DataAvailable(ptr)) break; @@ -1370,10 +1370,10 @@ const char* TcParser::MpFixed(PROTOBUF_TC_PARAM_DECL) { } // Copy the value: if (rep == field_layout::kRep64Bits) { - std::memcpy(Offset(msg, entry.offset), ptr, sizeof(uint64_t)); + RefAt(msg, entry.offset) = UnalignedLoad(ptr); ptr += sizeof(uint64_t); } else { - std::memcpy(Offset(msg, entry.offset), ptr, sizeof(uint32_t)); + RefAt(msg, entry.offset) = UnalignedLoad(ptr); ptr += sizeof(uint32_t); } PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); @@ -1401,7 +1401,7 @@ const char* TcParser::MpRepeatedFixed(PROTOBUF_TC_PARAM_DECL) { uint32_t next_tag; do { ptr = ptr2; - std::memcpy(field.Add(), ptr, size); + *field.Add() = UnalignedLoad(ptr); ptr += size; if (!ctx->DataAvailable(ptr)) break; ptr2 = ReadTag(ptr, &next_tag); @@ -1417,7 +1417,7 @@ const char* TcParser::MpRepeatedFixed(PROTOBUF_TC_PARAM_DECL) { uint32_t next_tag; do { ptr = ptr2; - std::memcpy(field.Add(), ptr, size); + *field.Add() = UnalignedLoad(ptr); ptr += size; if (!ctx->DataAvailable(ptr)) break; ptr2 = ReadTag(ptr, &next_tag); diff --git a/src/google/protobuf/io/coded_stream.h b/src/google/protobuf/io/coded_stream.h index 1024b40086..c8fc994f91 100644 --- a/src/google/protobuf/io/coded_stream.h +++ b/src/google/protobuf/io/coded_stream.h @@ -121,36 +121,13 @@ #include #include -#ifdef _WIN32 -// Assuming windows is always little-endian. -#if !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) -#define PROTOBUF_LITTLE_ENDIAN 1 -#endif #if defined(_MSC_VER) && _MSC_VER >= 1300 && !defined(__INTEL_COMPILER) // If MSVC has "/RTCc" set, it will complain about truncating casts at // runtime. This file contains some intentional truncating casts. #pragma runtime_checks("c", off) #endif -#else -#ifdef __APPLE__ -#include // __BYTE_ORDER -#elif defined(__FreeBSD__) -#include // __BYTE_ORDER -#elif (defined(sun) || defined(__sun)) && (defined(__SVR4) || defined(__svr4__)) -#include // __BYTE_ORDER -#elif defined(_AIX) || defined(__TOS_AIX__) -#include // BYTE_ORDER -#else -#if !defined(__QNX__) -#include // __BYTE_ORDER -#endif -#endif -#if ((defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)) || \ - (defined(__BYTE_ORDER) && __BYTE_ORDER == __LITTLE_ENDIAN)) && \ - !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) -#define PROTOBUF_LITTLE_ENDIAN 1 -#endif -#endif + + #include #include #include @@ -957,7 +934,8 @@ class PROTOBUF_EXPORT EpsCopyOutputStream { template uint8_t* WriteRawLittleEndian(const void* data, int size, uint8_t* ptr); -#ifndef PROTOBUF_LITTLE_ENDIAN +#if !defined(PROTOBUF_LITTLE_ENDIAN) || \ + defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) uint8_t* WriteRawLittleEndian32(const void* data, int size, uint8_t* ptr); uint8_t* WriteRawLittleEndian64(const void* data, int size, uint8_t* ptr); #endif @@ -1004,7 +982,8 @@ template <> inline uint8_t* EpsCopyOutputStream::WriteRawLittleEndian<4>(const void* data, int size, uint8_t* ptr) { -#ifdef PROTOBUF_LITTLE_ENDIAN +#if defined(PROTOBUF_LITTLE_ENDIAN) && \ + !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) return WriteRaw(data, size, ptr); #else return WriteRawLittleEndian32(data, size, ptr); @@ -1014,7 +993,8 @@ template <> inline uint8_t* EpsCopyOutputStream::WriteRawLittleEndian<8>(const void* data, int size, uint8_t* ptr) { -#ifdef PROTOBUF_LITTLE_ENDIAN +#if defined(PROTOBUF_LITTLE_ENDIAN) && \ + !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) return WriteRaw(data, size, ptr); #else return WriteRawLittleEndian64(data, size, ptr); @@ -1357,7 +1337,8 @@ inline bool CodedInputStream::ReadVarintSizeAsInt(int* value) { // static inline const uint8_t* CodedInputStream::ReadLittleEndian32FromArray( const uint8_t* buffer, uint32_t* value) { -#if defined(PROTOBUF_LITTLE_ENDIAN) +#if defined(PROTOBUF_LITTLE_ENDIAN) && \ + !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) memcpy(value, buffer, sizeof(*value)); return buffer + sizeof(*value); #else @@ -1371,7 +1352,8 @@ inline const uint8_t* CodedInputStream::ReadLittleEndian32FromArray( // static inline const uint8_t* CodedInputStream::ReadLittleEndian64FromArray( const uint8_t* buffer, uint64_t* value) { -#if defined(PROTOBUF_LITTLE_ENDIAN) +#if defined(PROTOBUF_LITTLE_ENDIAN) && \ + !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) memcpy(value, buffer, sizeof(*value)); return buffer + sizeof(*value); #else @@ -1389,7 +1371,8 @@ inline const uint8_t* CodedInputStream::ReadLittleEndian64FromArray( } inline bool CodedInputStream::ReadLittleEndian32(uint32_t* value) { -#if defined(PROTOBUF_LITTLE_ENDIAN) +#if defined(PROTOBUF_LITTLE_ENDIAN) && \ + !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) if (PROTOBUF_PREDICT_TRUE(BufferSize() >= static_cast(sizeof(*value)))) { buffer_ = ReadLittleEndian32FromArray(buffer_, value); return true; @@ -1402,7 +1385,8 @@ inline bool CodedInputStream::ReadLittleEndian32(uint32_t* value) { } inline bool CodedInputStream::ReadLittleEndian64(uint64_t* value) { -#if defined(PROTOBUF_LITTLE_ENDIAN) +#if defined(PROTOBUF_LITTLE_ENDIAN) && \ + !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) if (PROTOBUF_PREDICT_TRUE(BufferSize() >= static_cast(sizeof(*value)))) { buffer_ = ReadLittleEndian64FromArray(buffer_, value); return true; @@ -1689,7 +1673,8 @@ inline uint8_t* CodedOutputStream::WriteVarint32SignExtendedToArray( inline uint8_t* CodedOutputStream::WriteLittleEndian32ToArray(uint32_t value, uint8_t* target) { -#if defined(PROTOBUF_LITTLE_ENDIAN) +#if defined(PROTOBUF_LITTLE_ENDIAN) && \ + !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) memcpy(target, &value, sizeof(value)); #else target[0] = static_cast(value); @@ -1702,7 +1687,8 @@ inline uint8_t* CodedOutputStream::WriteLittleEndian32ToArray(uint32_t value, inline uint8_t* CodedOutputStream::WriteLittleEndian64ToArray(uint64_t value, uint8_t* target) { -#if defined(PROTOBUF_LITTLE_ENDIAN) +#if defined(PROTOBUF_LITTLE_ENDIAN) && \ + !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST) memcpy(target, &value, sizeof(value)); #else uint32_t part0 = static_cast(value); diff --git a/src/google/protobuf/message_unittest.inc b/src/google/protobuf/message_unittest.inc index d655dd49a5..d2ef2d96d4 100644 --- a/src/google/protobuf/message_unittest.inc +++ b/src/google/protobuf/message_unittest.inc @@ -278,10 +278,8 @@ TEST(MESSAGE_TEST_NAME, MergeFromUninitialized) { payload->mutable_optional_message()->set_dummy4(200); ASSERT_TRUE(p.ParsePartialFromString(o.SerializePartialAsString())); - GOOGLE_LOG(ERROR) << "seongkim: copy 1"; q.mutable_child()->set_dummy(500); q = p; - GOOGLE_LOG(ERROR) << "seongkim: copy 1 done"; q.ParsePartialFromString(q.SerializePartialAsString()); EXPECT_TRUE(TestUtil::EqualsToSerialized(q, o.SerializePartialAsString())); EXPECT_TRUE(TestUtil::EqualsToSerialized(q, p.SerializePartialAsString())); diff --git a/src/google/protobuf/parse_context.cc b/src/google/protobuf/parse_context.cc index edc8c5fa24..59852fdb7f 100644 --- a/src/google/protobuf/parse_context.cc +++ b/src/google/protobuf/parse_context.cc @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -234,19 +235,6 @@ const char* EpsCopyInputStream::AppendStringFallback(const char* ptr, int size, } -template -void byteswap(void* p); -template <> -void byteswap<1>(void* /*p*/) {} -template <> -void byteswap<4>(void* p) { - *static_cast(p) = bswap_32(*static_cast(p)); -} -template <> -void byteswap<8>(void* p) { - *static_cast(p) = bswap_64(*static_cast(p)); -} - const char* EpsCopyInputStream::InitFrom(io::ZeroCopyInputStream* zcis) { zcis_ = zcis; const void* data; diff --git a/src/google/protobuf/parse_context.h b/src/google/protobuf/parse_context.h index 80a7b42298..7c021c4e09 100644 --- a/src/google/protobuf/parse_context.h +++ b/src/google/protobuf/parse_context.h @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -486,10 +487,7 @@ struct EndianHelper<2> { static uint16_t Load(const void* p) { uint16_t tmp; std::memcpy(&tmp, p, 2); -#ifndef PROTOBUF_LITTLE_ENDIAN - tmp = bswap_16(tmp); -#endif - return tmp; + return little_endian::ToHost(tmp); } }; @@ -498,10 +496,7 @@ struct EndianHelper<4> { static uint32_t Load(const void* p) { uint32_t tmp; std::memcpy(&tmp, p, 4); -#ifndef PROTOBUF_LITTLE_ENDIAN - tmp = bswap_32(tmp); -#endif - return tmp; + return little_endian::ToHost(tmp); } }; @@ -510,10 +505,7 @@ struct EndianHelper<8> { static uint64_t Load(const void* p) { uint64_t tmp; std::memcpy(&tmp, p, 8); -#ifndef PROTOBUF_LITTLE_ENDIAN - tmp = bswap_64(tmp); -#endif - return tmp; + return little_endian::ToHost(tmp); } }; @@ -584,6 +576,82 @@ inline const char* ReadTag(const char* p, uint32_t* out, return tmp.first; } +// As above, but optimized to consume very few registers while still being fast, +// ReadTagInlined is useful for callers that don't mind the extra code but would +// like to avoid an extern function call causing spills into the stack. +// +// Two support routines for ReadTagInlined come first... +template +PROTOBUF_NODISCARD PROTOBUF_ALWAYS_INLINE constexpr T RotateLeft( + T x, int s) noexcept { + return static_cast(x << (s & (std::numeric_limits::digits - 1))) | + static_cast(x >> ((-s) & (std::numeric_limits::digits - 1))); +} + +PROTOBUF_NODISCARD inline PROTOBUF_ALWAYS_INLINE uint64_t +RotRight7AndReplaceLowByte(uint64_t res, const char& byte) { +#if defined(__x86_64__) && defined(__GNUC__) + // This will only use one register for `res`. + // `byte` comes as a reference to allow the compiler to generate code like: + // + // rorq $7, %rcx + // movb 1(%rax), %cl + // + // which avoids loading the incoming bytes into a separate register first. + asm("ror $7,%0\n\t" + "movb %1,%b0" + : "+r"(res) + : "m"(byte)); +#else + res = RotateLeft(res, -7); + res = res & ~0xFF; + res |= 0xFF & byte; +#endif + return res; +}; + +inline PROTOBUF_ALWAYS_INLINE +const char* ReadTagInlined(const char* ptr, uint32_t* out) { + uint64_t res = 0xFF & ptr[0]; + if (PROTOBUF_PREDICT_FALSE(res >= 128)) { + res = RotRight7AndReplaceLowByte(res, ptr[1]); + if (PROTOBUF_PREDICT_FALSE(res & 0x80)) { + res = RotRight7AndReplaceLowByte(res, ptr[2]); + if (PROTOBUF_PREDICT_FALSE(res & 0x80)) { + res = RotRight7AndReplaceLowByte(res, ptr[3]); + if (PROTOBUF_PREDICT_FALSE(res & 0x80)) { + // Note: this wouldn't work if res were 32-bit, + // because then replacing the low byte would overwrite + // the bottom 4 bits of the result. + res = RotRight7AndReplaceLowByte(res, ptr[4]); + if (PROTOBUF_PREDICT_FALSE(res & 0x80)) { + // The proto format does not permit longer than 5-byte encodings for + // tags. + *out = 0; + return nullptr; + } + *out = RotateLeft(res, 28); +#if defined(__GNUC__) + // Note: this asm statement prevents the compiler from + // trying to share the "return ptr + constant" among all + // branches. + asm("" : "+r"(ptr)); +#endif + return ptr + 5; + } + *out = RotateLeft(res, 21); + return ptr + 4; + } + *out = RotateLeft(res, 14); + return ptr + 3; + } + *out = RotateLeft(res, 7); + return ptr + 2; + } + *out = res; + return ptr + 1; +} + // Decode 2 consecutive bytes of a varint and returns the value, shifted left // by 1. It simultaneous updates *ptr to *ptr + 1 or *ptr + 2 depending if the // first byte's continuation bit is set. diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index d2e9f9c9c7..14185afe48 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -118,6 +118,27 @@ #define PROTOBUF_has_builtin_DEFINED_ #endif +// Portable PROTOBUF_BUILTIN_BSWAPxx definitions +// Code must check for availability, e.g.: `defined(PROTOBUF_BUILTIN_BSWAP32)` +#ifdef PROTOBUF_BUILTIN_BSWAP16 +#error PROTOBUF_BUILTIN_BSWAP16 was previously defined +#endif +#ifdef PROTOBUF_BUILTIN_BSWAP32 +#error PROTOBUF_BUILTIN_BSWAP32 was previously defined +#endif +#ifdef PROTOBUF_BUILTIN_BSWAP64 +#error PROTOBUF_BUILTIN_BSWAP64 was previously defined +#endif +#if defined(__GNUC__) || __has_builtin(__builtin_bswap16) +#define PROTOBUF_BUILTIN_BSWAP16(x) __builtin_bswap16(x) +#endif +#if defined(__GNUC__) || __has_builtin(__builtin_bswap32) +#define PROTOBUF_BUILTIN_BSWAP32(x) __builtin_bswap32(x) +#endif +#if defined(__GNUC__) || __has_builtin(__builtin_bswap64) +#define PROTOBUF_BUILTIN_BSWAP64(x) __builtin_bswap64(x) +#endif + // Portable check for GCC minimum version: // https://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html #if defined(__GNUC__) && defined(__GNUC_MINOR__) \ @@ -586,6 +607,22 @@ #define PROTOBUF_THREAD_LOCAL __thread #endif +// TODO(b/228173843): cleanup PROTOBUF_LITTLE_ENDIAN in various 3p forks. +#if (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \ + __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +#define PROTOBUF_LITTLE_ENDIAN 1 +#ifdef PROTOBUF_BIG_ENDIAN +#error Conflicting PROTOBUF_BIG_ENDIAN was previously defined +#endif +#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && \ + __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define PROTOBUF_BIG_ENDIAN 1 +#elif defined(_WIN32) || defined(__x86_64__) || defined(__aarch64__) +#define PROTOBUF_LITTLE_ENDIAN 1 +#else +#error "endian detection failed for current compiler" +#endif + // For enabling message owned arena, one major blocker is semantic change from // moving to copying when there is ownership transfer (e.g., move ctor, swap, // set allocated, release). This change not only causes performance regression diff --git a/src/google/protobuf/port_undef.inc b/src/google/protobuf/port_undef.inc index c7a102ffcc..6cb8eacc24 100644 --- a/src/google/protobuf/port_undef.inc +++ b/src/google/protobuf/port_undef.inc @@ -34,6 +34,10 @@ #ifndef PROTOBUF_NAMESPACE #error "port_undef.inc must be included after port_def.inc" #endif + +#undef PROTOBUF_BUILTIN_BSWAP16 +#undef PROTOBUF_BUILTIN_BSWAP32 +#undef PROTOBUF_BUILTIN_BSWAP64 #undef PROTOBUF_GNUC_MIN #undef PROTOBUF_MSC_VER_MIN #undef PROTOBUF_CPLUSPLUS_MIN @@ -79,6 +83,8 @@ #undef PROTOBUF_FINAL #undef PROTOBUF_FUTURE_FINAL #undef PROTOBUF_THREAD_LOCAL +#undef PROTOBUF_LITTLE_ENDIAN +#undef PROTOBUF_BIG_ENDIAN #undef PROTOBUF_MESSAGE_OWNED_ARENA_EXPERIMENT #undef PROTOBUF_CONSTINIT #undef PROTOBUF_CONSTEXPR diff --git a/src/google/protobuf/proto3_arena_unittest.cc b/src/google/protobuf/proto3_arena_unittest.cc index 7e266b9819..cfbe198abe 100644 --- a/src/google/protobuf/proto3_arena_unittest.cc +++ b/src/google/protobuf/proto3_arena_unittest.cc @@ -75,6 +75,7 @@ void SetAllFields(TestAllTypes* m) { m->set_optional_nested_enum(proto3_arena_unittest::TestAllTypes::BAZ); m->set_optional_foreign_enum(proto3_arena_unittest::FOREIGN_BAZ); m->mutable_optional_lazy_message()->set_bb(45); + m->mutable_optional_unverified_lazy_message()->set_bb(46); m->add_repeated_int32(100); m->add_repeated_string("asdf"); m->add_repeated_bytes("jkl;"); @@ -101,6 +102,8 @@ void ExpectAllFieldsSet(const TestAllTypes& m) { EXPECT_EQ(proto3_arena_unittest::FOREIGN_BAZ, m.optional_foreign_enum()); EXPECT_EQ(true, m.has_optional_lazy_message()); EXPECT_EQ(45, m.optional_lazy_message().bb()); + EXPECT_EQ(true, m.has_optional_unverified_lazy_message()); + EXPECT_EQ(46, m.optional_unverified_lazy_message().bb()); EXPECT_EQ(1, m.repeated_int32_size()); EXPECT_EQ(100, m.repeated_int32(0)); diff --git a/src/google/protobuf/proto3_lite_unittest.inc b/src/google/protobuf/proto3_lite_unittest.inc index 85f428a0a8..5878163b53 100644 --- a/src/google/protobuf/proto3_lite_unittest.inc +++ b/src/google/protobuf/proto3_lite_unittest.inc @@ -53,6 +53,7 @@ void SetAllFields(TestAllTypes* m) { m->set_optional_foreign_enum( UNITTEST::FOREIGN_BAZ); m->mutable_optional_lazy_message()->set_bb(45); + m->mutable_optional_unverified_lazy_message()->set_bb(46); m->add_repeated_int32(100); m->add_repeated_string("asdf"); m->add_repeated_bytes("jkl;"); @@ -81,6 +82,8 @@ void ExpectAllFieldsSet(const TestAllTypes& m) { m.optional_foreign_enum()); EXPECT_EQ(true, m.has_optional_lazy_message()); EXPECT_EQ(45, m.optional_lazy_message().bb()); + EXPECT_EQ(true, m.has_optional_unverified_lazy_message()); + EXPECT_EQ(46, m.optional_unverified_lazy_message().bb()); EXPECT_EQ(1, m.repeated_int32_size()); EXPECT_EQ(100, m.repeated_int32(0)); diff --git a/src/google/protobuf/reflection_ops.cc b/src/google/protobuf/reflection_ops.cc index b9f8e4e4a3..3a1972e274 100644 --- a/src/google/protobuf/reflection_ops.cc +++ b/src/google/protobuf/reflection_ops.cc @@ -170,8 +170,10 @@ void ReflectionOps::Merge(const Message& from, Message* to) { } } - to_reflection->MutableUnknownFields(to)->MergeFrom( - from_reflection->GetUnknownFields(from)); + if (!from_reflection->GetUnknownFields(from).empty()) { + to_reflection->MutableUnknownFields(to)->MergeFrom( + from_reflection->GetUnknownFields(from)); + } } void ReflectionOps::Clear(Message* message) { diff --git a/src/google/protobuf/test_util.h b/src/google/protobuf/test_util.h index 605f60b303..b18d7b676f 100644 --- a/src/google/protobuf/test_util.h +++ b/src/google/protobuf/test_util.h @@ -238,6 +238,10 @@ inline void TestUtil::ReflectionTester::SetAllFieldsViaReflection( sub_message = reflection->MutableMessage(message, F("optional_lazy_message")); sub_message->GetReflection()->SetInt32(sub_message, nested_b_, 127); + sub_message = reflection->MutableMessage( + message, F("optional_unverified_lazy_message")); + sub_message->GetReflection()->SetInt32(sub_message, nested_b_, 128); + // ----------------------------------------------------------------- reflection->AddInt32(message, F("repeated_int32"), 201); @@ -468,6 +472,8 @@ inline void TestUtil::ReflectionTester::ExpectAllFieldsSetViaReflection1( EXPECT_TRUE( reflection->HasField(message, F("optional_public_import_message"))); EXPECT_TRUE(reflection->HasField(message, F("optional_lazy_message"))); + EXPECT_TRUE( + reflection->HasField(message, F("optional_unverified_lazy_message"))); sub_message = &reflection->GetMessage(message, F("optionalgroup")); EXPECT_TRUE(sub_message->GetReflection()->HasField(*sub_message, group_a_)); @@ -482,6 +488,9 @@ inline void TestUtil::ReflectionTester::ExpectAllFieldsSetViaReflection1( EXPECT_TRUE(sub_message->GetReflection()->HasField(*sub_message, import_e_)); sub_message = &reflection->GetMessage(message, F("optional_lazy_message")); EXPECT_TRUE(sub_message->GetReflection()->HasField(*sub_message, nested_b_)); + sub_message = + &reflection->GetMessage(message, F("optional_unverified_lazy_message")); + EXPECT_TRUE(sub_message->GetReflection()->HasField(*sub_message, nested_b_)); EXPECT_TRUE(reflection->HasField(message, F("optional_nested_enum"))); EXPECT_TRUE(reflection->HasField(message, F("optional_foreign_enum"))); @@ -530,6 +539,10 @@ inline void TestUtil::ReflectionTester::ExpectAllFieldsSetViaReflection1( sub_message = &reflection->GetMessage(message, F("optional_lazy_message")); EXPECT_EQ(127, sub_message->GetReflection()->GetInt32(*sub_message, nested_b_)); + sub_message = + &reflection->GetMessage(message, F("optional_unverified_lazy_message")); + EXPECT_EQ(128, + sub_message->GetReflection()->GetInt32(*sub_message, nested_b_)); EXPECT_EQ(nested_baz_, reflection->GetEnum(message, F("optional_nested_enum"))); @@ -897,6 +910,8 @@ inline void TestUtil::ReflectionTester::ExpectClearViaReflection( EXPECT_FALSE( reflection->HasField(message, F("optional_public_import_message"))); EXPECT_FALSE(reflection->HasField(message, F("optional_lazy_message"))); + EXPECT_FALSE( + reflection->HasField(message, F("optional_unverified_lazy_message"))); EXPECT_FALSE(reflection->HasField(message, F("optional_nested_enum"))); EXPECT_FALSE(reflection->HasField(message, F("optional_foreign_enum"))); @@ -949,6 +964,10 @@ inline void TestUtil::ReflectionTester::ExpectClearViaReflection( sub_message = &reflection->GetMessage(message, F("optional_lazy_message")); EXPECT_FALSE(sub_message->GetReflection()->HasField(*sub_message, nested_b_)); EXPECT_EQ(0, sub_message->GetReflection()->GetInt32(*sub_message, nested_b_)); + sub_message = + &reflection->GetMessage(message, F("optional_unverified_lazy_message")); + EXPECT_FALSE(sub_message->GetReflection()->HasField(*sub_message, nested_b_)); + EXPECT_EQ(0, sub_message->GetReflection()->GetInt32(*sub_message, nested_b_)); // Enums without defaults are set to the first value in the enum. EXPECT_EQ(nested_foo_, diff --git a/src/google/protobuf/test_util.inc b/src/google/protobuf/test_util.inc index 8b7fa73a8e..8d44afadd5 100644 --- a/src/google/protobuf/test_util.inc +++ b/src/google/protobuf/test_util.inc @@ -143,6 +143,7 @@ inline void TestUtil::SetOptionalFields(UNITTEST::TestAllTypes* message) { message->mutable_optional_import_message()->set_d(120); message->mutable_optional_public_import_message()->set_e(126); message->mutable_optional_lazy_message()->set_bb(127); + message->mutable_optional_unverified_lazy_message()->set_bb(128); message->set_optional_nested_enum(UNITTEST::TestAllTypes::BAZ); message->set_optional_foreign_enum(UNITTEST::FOREIGN_BAZ); @@ -347,6 +348,7 @@ inline void TestUtil::ExpectAllFieldsSet( EXPECT_TRUE(message.has_optional_import_message()); EXPECT_TRUE(message.has_optional_public_import_message()); EXPECT_TRUE(message.has_optional_lazy_message()); + EXPECT_TRUE(message.has_optional_unverified_lazy_message()); EXPECT_TRUE(message.optionalgroup().has_a()); EXPECT_TRUE(message.optional_nested_message().has_bb()); @@ -354,6 +356,7 @@ inline void TestUtil::ExpectAllFieldsSet( EXPECT_TRUE(message.optional_import_message().has_d()); EXPECT_TRUE(message.optional_public_import_message().has_e()); EXPECT_TRUE(message.optional_lazy_message().has_bb()); + EXPECT_TRUE(message.optional_unverified_lazy_message().has_bb()); EXPECT_TRUE(message.has_optional_nested_enum()); EXPECT_TRUE(message.has_optional_foreign_enum()); @@ -386,6 +389,7 @@ inline void TestUtil::ExpectAllFieldsSet( EXPECT_EQ(120, message.optional_import_message().d()); EXPECT_EQ(126, message.optional_public_import_message().e()); EXPECT_EQ(127, message.optional_lazy_message().bb()); + EXPECT_EQ(128, message.optional_unverified_lazy_message().bb()); EXPECT_EQ(UNITTEST::TestAllTypes::BAZ, message.optional_nested_enum()); EXPECT_EQ(UNITTEST::FOREIGN_BAZ, message.optional_foreign_enum()); @@ -556,6 +560,7 @@ inline void TestUtil::ExpectClear(const UNITTEST::TestAllTypes& message) { EXPECT_FALSE(message.has_optional_import_message()); EXPECT_FALSE(message.has_optional_public_import_message()); EXPECT_FALSE(message.has_optional_lazy_message()); + EXPECT_FALSE(message.has_optional_unverified_lazy_message()); EXPECT_FALSE(message.has_optional_nested_enum()); EXPECT_FALSE(message.has_optional_foreign_enum()); @@ -588,6 +593,7 @@ inline void TestUtil::ExpectClear(const UNITTEST::TestAllTypes& message) { EXPECT_FALSE(message.optional_import_message().has_d()); EXPECT_FALSE(message.optional_public_import_message().has_e()); EXPECT_FALSE(message.optional_lazy_message().has_bb()); + EXPECT_FALSE(message.optional_unverified_lazy_message().has_bb()); EXPECT_EQ(0, message.optionalgroup().a()); EXPECT_EQ(0, message.optional_nested_message().bb()); @@ -595,6 +601,7 @@ inline void TestUtil::ExpectClear(const UNITTEST::TestAllTypes& message) { EXPECT_EQ(0, message.optional_import_message().d()); EXPECT_EQ(0, message.optional_public_import_message().e()); EXPECT_EQ(0, message.optional_lazy_message().bb()); + EXPECT_EQ(0, message.optional_unverified_lazy_message().bb()); // Enums without defaults are set to the first value in the enum. EXPECT_EQ(UNITTEST::TestAllTypes::FOO, message.optional_nested_enum()); @@ -987,6 +994,9 @@ inline void TestUtil::SetAllExtensions(UNITTEST::TestAllExtensions* message) { ->set_e(126); message->MutableExtension(UNITTEST::optional_lazy_message_extension) ->set_bb(127); + message + ->MutableExtension(UNITTEST::optional_unverified_lazy_message_extension) + ->set_bb(128); // ----------------------------------------------------------------- @@ -1183,6 +1193,8 @@ inline void TestUtil::ExpectAllExtensionsSet( EXPECT_TRUE( message.HasExtension(UNITTEST::optional_public_import_message_extension)); EXPECT_TRUE(message.HasExtension(UNITTEST::optional_lazy_message_extension)); + EXPECT_TRUE(message.HasExtension( + UNITTEST::optional_unverified_lazy_message_extension)); EXPECT_TRUE(message.GetExtension(UNITTEST::optionalgroup_extension).has_a()); EXPECT_TRUE(message.GetExtension(UNITTEST::optional_nested_message_extension) @@ -1196,6 +1208,9 @@ inline void TestUtil::ExpectAllExtensionsSet( .has_e()); EXPECT_TRUE( message.GetExtension(UNITTEST::optional_lazy_message_extension).has_bb()); + EXPECT_TRUE( + message.GetExtension(UNITTEST::optional_unverified_lazy_message_extension) + .has_bb()); EXPECT_TRUE(message.HasExtension(UNITTEST::optional_nested_enum_extension)); EXPECT_TRUE(message.HasExtension(UNITTEST::optional_foreign_enum_extension)); @@ -1248,6 +1263,10 @@ inline void TestUtil::ExpectAllExtensionsSet( EXPECT_EQ( 127, message.GetExtension(UNITTEST::optional_lazy_message_extension).bb()); + EXPECT_EQ( + 128, + message.GetExtension(UNITTEST::optional_unverified_lazy_message_extension) + .bb()); // ----------------------------------------------------------------- @@ -1476,6 +1495,8 @@ inline void TestUtil::ExpectExtensionsClear( EXPECT_FALSE( message.HasExtension(UNITTEST::optional_public_import_message_extension)); EXPECT_FALSE(message.HasExtension(UNITTEST::optional_lazy_message_extension)); + EXPECT_FALSE(message.HasExtension( + UNITTEST::optional_unverified_lazy_message_extension)); EXPECT_FALSE(message.HasExtension(UNITTEST::optional_nested_enum_extension)); EXPECT_FALSE(message.HasExtension(UNITTEST::optional_foreign_enum_extension)); @@ -1515,6 +1536,9 @@ inline void TestUtil::ExpectExtensionsClear( .has_e()); EXPECT_FALSE( message.GetExtension(UNITTEST::optional_lazy_message_extension).has_bb()); + EXPECT_FALSE( + message.GetExtension(UNITTEST::optional_unverified_lazy_message_extension) + .has_bb()); EXPECT_EQ(0, message.GetExtension(UNITTEST::optionalgroup_extension).a()); EXPECT_EQ( @@ -1531,6 +1555,10 @@ inline void TestUtil::ExpectExtensionsClear( .e()); EXPECT_EQ( 0, message.GetExtension(UNITTEST::optional_lazy_message_extension).bb()); + EXPECT_EQ( + 0, + message.GetExtension(UNITTEST::optional_unverified_lazy_message_extension) + .bb()); // Enums without defaults are set to the first value in the enum. EXPECT_EQ(UNITTEST::TestAllTypes::FOO, diff --git a/src/google/protobuf/test_util_lite.cc b/src/google/protobuf/test_util_lite.cc index e7eb60a6b7..628f1a0294 100644 --- a/src/google/protobuf/test_util_lite.cc +++ b/src/google/protobuf/test_util_lite.cc @@ -65,6 +65,7 @@ void TestUtilLite::SetAllFields(unittest::TestAllTypesLite* message) { message->mutable_optional_import_message()->set_d(120); message->mutable_optional_public_import_message()->set_e(126); message->mutable_optional_lazy_message()->set_bb(127); + message->mutable_optional_unverified_lazy_message()->set_bb(128); message->set_optional_nested_enum(unittest::TestAllTypesLite::BAZ); message->set_optional_foreign_enum(unittest::FOREIGN_LITE_BAZ); @@ -214,6 +215,7 @@ void TestUtilLite::ExpectAllFieldsSet( EXPECT_TRUE(message.has_optional_import_message()); EXPECT_TRUE(message.has_optional_public_import_message()); EXPECT_TRUE(message.has_optional_lazy_message()); + EXPECT_TRUE(message.has_optional_unverified_lazy_message()); EXPECT_TRUE(message.optionalgroup().has_a()); EXPECT_TRUE(message.optional_nested_message().has_bb()); @@ -221,6 +223,7 @@ void TestUtilLite::ExpectAllFieldsSet( EXPECT_TRUE(message.optional_import_message().has_d()); EXPECT_TRUE(message.optional_public_import_message().has_e()); EXPECT_TRUE(message.optional_lazy_message().has_bb()); + EXPECT_TRUE(message.optional_unverified_lazy_message().has_bb()); EXPECT_TRUE(message.has_optional_nested_enum()); EXPECT_TRUE(message.has_optional_foreign_enum()); @@ -249,6 +252,7 @@ void TestUtilLite::ExpectAllFieldsSet( EXPECT_EQ(120, message.optional_import_message().d()); EXPECT_EQ(126, message.optional_public_import_message().e()); EXPECT_EQ(127, message.optional_lazy_message().bb()); + EXPECT_EQ(128, message.optional_unverified_lazy_message().bb()); EXPECT_EQ(unittest::TestAllTypesLite::BAZ, message.optional_nested_enum()); EXPECT_EQ(unittest::FOREIGN_LITE_BAZ, message.optional_foreign_enum()); @@ -415,6 +419,7 @@ void TestUtilLite::ExpectClear(const unittest::TestAllTypesLite& message) { EXPECT_FALSE(message.has_optional_import_message()); EXPECT_FALSE(message.has_optional_public_import_message()); EXPECT_FALSE(message.has_optional_lazy_message()); + EXPECT_FALSE(message.has_optional_unverified_lazy_message()); EXPECT_FALSE(message.has_optional_nested_enum()); EXPECT_FALSE(message.has_optional_foreign_enum()); @@ -445,6 +450,7 @@ void TestUtilLite::ExpectClear(const unittest::TestAllTypesLite& message) { EXPECT_FALSE(message.optional_import_message().has_d()); EXPECT_FALSE(message.optional_public_import_message().has_e()); EXPECT_FALSE(message.optional_lazy_message().has_bb()); + EXPECT_FALSE(message.optional_unverified_lazy_message().has_bb()); EXPECT_EQ(0, message.optionalgroup().a()); EXPECT_EQ(0, message.optional_nested_message().bb()); @@ -836,6 +842,10 @@ void TestUtilLite::SetAllExtensions(unittest::TestAllExtensionsLite* message) { ->set_e(126); message->MutableExtension(unittest::optional_lazy_message_extension_lite) ->set_bb(127); + message + ->MutableExtension( + unittest::optional_unverified_lazy_message_extension_lite) + ->set_bb(128); message->SetExtension(unittest::optional_nested_enum_extension_lite, unittest::TestAllTypesLite::BAZ); @@ -1022,6 +1032,8 @@ void TestUtilLite::ExpectAllExtensionsSet( unittest::optional_public_import_message_extension_lite)); EXPECT_TRUE( message.HasExtension(unittest::optional_lazy_message_extension_lite)); + EXPECT_TRUE(message.HasExtension( + unittest::optional_unverified_lazy_message_extension_lite)); EXPECT_TRUE( message.GetExtension(unittest::optionalgroup_extension_lite).has_a()); @@ -1041,6 +1053,10 @@ void TestUtilLite::ExpectAllExtensionsSet( EXPECT_TRUE( message.GetExtension(unittest::optional_lazy_message_extension_lite) .has_bb()); + EXPECT_TRUE(message + .GetExtension( + unittest::optional_unverified_lazy_message_extension_lite) + .has_bb()); EXPECT_TRUE( message.HasExtension(unittest::optional_nested_enum_extension_lite)); @@ -1099,6 +1115,11 @@ void TestUtilLite::ExpectAllExtensionsSet( EXPECT_EQ(127, message.GetExtension(unittest::optional_lazy_message_extension_lite) .bb()); + EXPECT_EQ(128, + message + .GetExtension( + unittest::optional_unverified_lazy_message_extension_lite) + .bb()); EXPECT_EQ( unittest::TestAllTypesLite::BAZ, diff --git a/src/google/protobuf/testdata/golden_message b/src/google/protobuf/testdata/golden_message index 0b7e6552c6..5825975ce0 100644 Binary files a/src/google/protobuf/testdata/golden_message and b/src/google/protobuf/testdata/golden_message differ diff --git a/src/google/protobuf/testdata/golden_message_oneof_implemented b/src/google/protobuf/testdata/golden_message_oneof_implemented index b48c898526..794ca5e0d1 100644 Binary files a/src/google/protobuf/testdata/golden_message_oneof_implemented and b/src/google/protobuf/testdata/golden_message_oneof_implemented differ diff --git a/src/google/protobuf/testdata/text_format_unittest_data_oneof_implemented.txt b/src/google/protobuf/testdata/text_format_unittest_data_oneof_implemented.txt index ec95e1e81a..86389c93cb 100644 --- a/src/google/protobuf/testdata/text_format_unittest_data_oneof_implemented.txt +++ b/src/google/protobuf/testdata/text_format_unittest_data_oneof_implemented.txt @@ -36,6 +36,9 @@ optional_public_import_message { optional_lazy_message { bb: 127 } +optional_unverified_lazy_message { + bb: 128 +} repeated_int32: 201 repeated_int32: 301 repeated_int64: 202 diff --git a/src/google/protobuf/testdata/text_format_unittest_data_pointy.txt b/src/google/protobuf/testdata/text_format_unittest_data_pointy.txt index e1011ebf15..788025c515 100644 --- a/src/google/protobuf/testdata/text_format_unittest_data_pointy.txt +++ b/src/google/protobuf/testdata/text_format_unittest_data_pointy.txt @@ -36,6 +36,9 @@ optional_public_import_message < optional_lazy_message < bb: 127 > +optional_unverified_lazy_message < + bb: 128 +> repeated_int32: 201 repeated_int32: 301 repeated_int64: 202 diff --git a/src/google/protobuf/testdata/text_format_unittest_data_pointy_oneof.txt b/src/google/protobuf/testdata/text_format_unittest_data_pointy_oneof.txt index 95109f62ab..b2d3367098 100644 --- a/src/google/protobuf/testdata/text_format_unittest_data_pointy_oneof.txt +++ b/src/google/protobuf/testdata/text_format_unittest_data_pointy_oneof.txt @@ -36,6 +36,9 @@ optional_public_import_message < optional_lazy_message < bb: 127 > +optional_unverified_lazy_message < + bb: 128 +> repeated_int32: 201 repeated_int32: 301 repeated_int64: 202 diff --git a/src/google/protobuf/testdata/text_format_unittest_extensions_data.txt b/src/google/protobuf/testdata/text_format_unittest_extensions_data.txt index 8c8b1eb4d8..5c3a03ac37 100644 --- a/src/google/protobuf/testdata/text_format_unittest_extensions_data.txt +++ b/src/google/protobuf/testdata/text_format_unittest_extensions_data.txt @@ -36,6 +36,9 @@ [protobuf_unittest.optional_lazy_message_extension] { bb: 127 } +[protobuf_unittest.optional_unverified_lazy_message_extension] { + bb: 128 +} [protobuf_unittest.repeated_int32_extension]: 201 [protobuf_unittest.repeated_int32_extension]: 301 [protobuf_unittest.repeated_int64_extension]: 202 diff --git a/src/google/protobuf/testdata/text_format_unittest_extensions_data_pointy.txt b/src/google/protobuf/testdata/text_format_unittest_extensions_data_pointy.txt index 132f7445f3..4233ca78f3 100644 --- a/src/google/protobuf/testdata/text_format_unittest_extensions_data_pointy.txt +++ b/src/google/protobuf/testdata/text_format_unittest_extensions_data_pointy.txt @@ -36,6 +36,9 @@ [protobuf_unittest.optional_lazy_message_extension] < bb: 127 > +[protobuf_unittest.optional_unverified_lazy_message_extension] < + bb: 128 +> [protobuf_unittest.repeated_int32_extension]: 201 [protobuf_unittest.repeated_int32_extension]: 301 [protobuf_unittest.repeated_int64_extension]: 202 diff --git a/src/google/protobuf/unittest.proto b/src/google/protobuf/unittest.proto index f009ce7f7a..44f8a4861b 100644 --- a/src/google/protobuf/unittest.proto +++ b/src/google/protobuf/unittest.proto @@ -113,6 +113,7 @@ message TestAllTypes { optional_public_import_message = 26; optional NestedMessage optional_lazy_message = 27 [lazy=true]; + optional NestedMessage optional_unverified_lazy_message = 28 [unverified_lazy=true]; // Repeated repeated int32 repeated_int32 = 31; @@ -264,6 +265,8 @@ extend TestAllExtensions { optional TestAllTypes.NestedMessage optional_lazy_message_extension = 27 [lazy=true]; + optional TestAllTypes.NestedMessage + optional_unverified_lazy_message_extension = 28 [unverified_lazy=true]; // Repeated repeated int32 repeated_int32_extension = 31; diff --git a/src/google/protobuf/unittest_lite.proto b/src/google/protobuf/unittest_lite.proto index e2730acdbf..010d4a9c0b 100644 --- a/src/google/protobuf/unittest_lite.proto +++ b/src/google/protobuf/unittest_lite.proto @@ -97,6 +97,8 @@ message TestAllTypesLite { optional_public_import_message = 26; optional NestedMessage optional_lazy_message = 27 [lazy = true]; + optional NestedMessage optional_unverified_lazy_message = 28 + [unverified_lazy = true]; // Repeated repeated int32 repeated_int32 = 31; @@ -247,6 +249,9 @@ extend TestAllExtensionsLite { optional TestAllTypesLite.NestedMessage optional_lazy_message_extension_lite = 27 [lazy = true]; + optional TestAllTypesLite.NestedMessage + optional_unverified_lazy_message_extension_lite = 28 + [unverified_lazy = true]; // Repeated repeated int32 repeated_int32_extension_lite = 31; @@ -406,7 +411,9 @@ message TestEmptyMessageWithExtensionsLite { extensions 1 to max; } -enum V1EnumLite { V1_FIRST = 1; } +enum V1EnumLite { + V1_FIRST = 1; +} enum V2EnumLite { V2_FIRST = 1; diff --git a/src/google/protobuf/unittest_proto3.proto b/src/google/protobuf/unittest_proto3.proto index 8b78075290..910f401019 100644 --- a/src/google/protobuf/unittest_proto3.proto +++ b/src/google/protobuf/unittest_proto3.proto @@ -96,6 +96,7 @@ message TestAllTypes { 26; NestedMessage optional_lazy_message = 27 [lazy = true]; + NestedMessage optional_unverified_lazy_message = 28 [unverified_lazy = true]; protobuf_unittest_import.ImportMessage optional_lazy_import_message = 115 [lazy = true]; diff --git a/src/google/protobuf/unittest_proto3_arena.proto b/src/google/protobuf/unittest_proto3_arena.proto index 1752939797..7dc6cd066d 100644 --- a/src/google/protobuf/unittest_proto3_arena.proto +++ b/src/google/protobuf/unittest_proto3_arena.proto @@ -96,6 +96,7 @@ message TestAllTypes { optional_public_import_message = 26; NestedMessage optional_lazy_message = 27 [lazy=true]; + NestedMessage optional_unverified_lazy_message = 28 [unverified_lazy=true]; protobuf_unittest_import.ImportMessage optional_lazy_import_message = 115 [lazy = true]; diff --git a/src/google/protobuf/util/field_mask_util_test.cc b/src/google/protobuf/util/field_mask_util_test.cc index df5d087379..bcad739fc1 100644 --- a/src/google/protobuf/util/field_mask_util_test.cc +++ b/src/google/protobuf/util/field_mask_util_test.cc @@ -227,7 +227,7 @@ TEST(FieldMaskUtilTest, TestGetFieldMaskForAllFields) { EXPECT_TRUE(FieldMaskUtil::IsPathInFieldMask("bb", mask)); mask = FieldMaskUtil::GetFieldMaskForAllFields(); - EXPECT_EQ(75, mask.paths_size()); + EXPECT_EQ(76, mask.paths_size()); EXPECT_TRUE(FieldMaskUtil::IsPathInFieldMask("optional_int32", mask)); EXPECT_TRUE(FieldMaskUtil::IsPathInFieldMask("optional_int64", mask)); EXPECT_TRUE(FieldMaskUtil::IsPathInFieldMask("optional_uint32", mask));