diff --git a/.gitignore b/.gitignore index b9d795ce4c..1040236af1 100644 --- a/.gitignore +++ b/.gitignore @@ -215,5 +215,10 @@ _build/ .idea *.iml +# Eclipse +**/.settings +**/.project +**/.classpath + # BenchmarkDotNet BenchmarkDotNet.Artifacts/ diff --git a/BUILD b/BUILD index 8a84954c64..887e5e3845 100644 --- a/BUILD +++ b/BUILD @@ -146,6 +146,7 @@ load( "adapt_proto_library", "cc_proto_library", "internal_copied_filegroup", + "internal_gen_kt_protos", "internal_gen_well_known_protos_java", "internal_protobuf_py_tests", "py_proto_library", @@ -891,6 +892,23 @@ internal_gen_well_known_protos_java( deps = [proto + "_proto" for proto in LITE_WELL_KNOWN_PROTO_MAP.keys()], ) +internal_gen_kt_protos( + name = "gen_well_known_protos_kotlin", + visibility = [ + "//java:__subpackages__", + ], + deps = [proto + "_proto" for proto in WELL_KNOWN_PROTO_MAP.keys()], +) + +internal_gen_kt_protos( + name = "gen_well_known_protos_kotlinlite", + visibility = [ + "//java:__subpackages__", + ], + lite = True, + deps = [proto + "_proto" for proto in LITE_WELL_KNOWN_PROTO_MAP.keys()], +) + alias( name = "protobuf_java", actual = "//java/core", @@ -1428,143 +1446,60 @@ filegroup( # Kotlin proto rules -genrule( - name = "gen_kotlin_unittest_lite", +proto_library( + name = "kt_unittest_lite", srcs = [ "src/google/protobuf/unittest_lite.proto", "src/google/protobuf/unittest_import_lite.proto", "src/google/protobuf/unittest_import_public_lite.proto", "src/google/protobuf/map_lite_unittest.proto", ], - outs = [ - "TestAllTypesLiteKt.kt", - "ForeignMessageLiteKt.kt", - "TestAllExtensionsLiteKt.kt", - "TestEmptyMessageLiteKt.kt", - "TestEmptyMessageWithExtensionsLiteKt.kt", - "TestMapLiteKt.kt", - "OptionalGroup_extension_liteKt.kt", - "RepeatedGroup_extension_liteKt.kt", - ], + strip_import_prefix = "src", +) + +internal_gen_kt_protos( + name = "gen_kotlin_unittest_lite", + deps = [":kt_unittest_lite"], + lite = True, visibility = ["//java:__subpackages__"], - cmd = "$(location //:protoc) " + - "--kotlin_out=lite:$(@D) -Isrc/ " + - "$(locations src/google/protobuf/unittest_lite.proto) && " + - "$(location //:protoc) " + - "--kotlin_out=lite:$(@D) -Isrc/ " + - "$(locations src/google/protobuf/map_lite_unittest.proto) && " + - "cp $(@D)/com/google/protobuf/TestAllTypesLiteKt.kt " + - "$(location TestAllTypesLiteKt.kt) && " + - "cp $(@D)/com/google/protobuf/ForeignMessageLiteKt.kt " + - "$(location ForeignMessageLiteKt.kt) && " + - "cp $(@D)/com/google/protobuf/TestAllExtensionsLiteKt.kt " + - "$(location TestAllExtensionsLiteKt.kt) && " + - "cp $(@D)/com/google/protobuf/TestAllTypesLiteKt.kt " + - "$(location TestAllTypesLiteKt.kt) && " + - "cp $(@D)/com/google/protobuf/TestEmptyMessageLiteKt.kt " + - "$(location TestEmptyMessageLiteKt.kt) && " + - "cp $(@D)/com/google/protobuf/TestEmptyMessageWithExtensionsLiteKt.kt " + - "$(location TestEmptyMessageWithExtensionsLiteKt.kt) && " + - "cp $(@D)/protobuf_unittest/TestMapLiteKt.kt " + - "$(location TestMapLiteKt.kt) && " + - "cp $(@D)/com/google/protobuf/OptionalGroup_extension_liteKt.kt " + - "$(location OptionalGroup_extension_liteKt.kt) && " + - "cp $(@D)/com/google/protobuf/RepeatedGroup_extension_liteKt.kt " + - "$(location RepeatedGroup_extension_liteKt.kt)", - tools = [":protoc"], ) -genrule( - name = "gen_kotlin_unittest", +proto_library( + name = "kt_unittest", srcs = [ "src/google/protobuf/unittest.proto", "src/google/protobuf/unittest_import.proto", "src/google/protobuf/unittest_import_public.proto", "src/google/protobuf/map_proto2_unittest.proto", ], - outs = [ - "TestAllTypesKt.kt", - "ForeignMessageKt.kt", - "TestAllExtensionsKt.kt", - "TestEmptyMessageKt.kt", - "TestEmptyMessageWithExtensionsKt.kt", - "TestIntIntMapKt.kt", - "TestEnumMapKt.kt", - "TestMapsKt.kt", - "OptionalGroup_extensionKt.kt", - "RepeatedGroup_extensionKt.kt", - ], - visibility = ["//java:__subpackages__"], - cmd = "$(location //:protoc) " + - "--kotlin_out=shared,immutable:$(@D) -Isrc/ " + - "$(location src/google/protobuf/unittest.proto) && " + - "$(location //:protoc) " + - "--kotlin_out=shared,immutable:$(@D) -Isrc/ " + - "$(location src/google/protobuf/map_proto2_unittest.proto) && " + - "cp $(@D)/protobuf_unittest/TestAllTypesKt.kt " + - "$(location TestAllTypesKt.kt) && " + - "cp $(@D)/protobuf_unittest/ForeignMessageKt.kt " + - "$(location ForeignMessageKt.kt) && " + - "cp $(@D)/protobuf_unittest/TestAllExtensionsKt.kt " + - "$(location TestAllExtensionsKt.kt) && " + - "cp $(@D)/protobuf_unittest/TestEmptyMessageKt.kt " + - "$(location TestEmptyMessageKt.kt) && " + - "cp $(@D)/protobuf_unittest/TestEmptyMessageWithExtensionsKt.kt " + - "$(location TestEmptyMessageWithExtensionsKt.kt) && " + - "cp $(@D)/protobuf_unittest/TestIntIntMapKt.kt " + - "$(location TestIntIntMapKt.kt) && " + - "cp $(@D)/protobuf_unittest/TestEnumMapKt.kt " + - "$(location TestEnumMapKt.kt) && " + - "cp $(@D)/protobuf_unittest/TestMapsKt.kt " + - "$(location TestMapsKt.kt) && " + - "cp $(@D)/protobuf_unittest/OptionalGroup_extensionKt.kt " + - "$(location OptionalGroup_extensionKt.kt) && " + - "cp $(@D)/protobuf_unittest/RepeatedGroup_extensionKt.kt " + - "$(location RepeatedGroup_extensionKt.kt)", - tools = ["//:protoc"], + strip_import_prefix = "src", ) -genrule( - name = "gen_kotlin_proto3_unittest_lite", - srcs = [ - "src/google/protobuf/unittest_proto3_lite.proto", - "src/google/protobuf/unittest_import.proto", - "src/google/protobuf/unittest_import_public.proto", - ], - outs = [ - "TestAllTypesProto3LiteKt.kt", - "TestEmptyMessageProto3LiteKt.kt", - ], +internal_gen_kt_protos( + name = "gen_kotlin_unittest", + deps = [":kt_unittest"], visibility = ["//java:__subpackages__"], - cmd = "$(location //:protoc) " + - "--kotlin_out=lite:$(@D) -Isrc/ " + - "$(location src/google/protobuf/unittest_proto3_lite.proto) && " + - "cp $(@D)/proto3_lite_unittest/TestAllTypesKt.kt " + - "$(location TestAllTypesProto3LiteKt.kt) && " + - "cp $(@D)/proto3_lite_unittest/TestEmptyMessageKt.kt " + - "$(location TestEmptyMessageProto3LiteKt.kt)", - tools = ["//:protoc"], ) -genrule( - name = "gen_kotlin_proto3_unittest", +proto_library( + name = "kt_proto3_unittest", srcs = [ "src/google/protobuf/unittest_proto3.proto", "src/google/protobuf/unittest_import.proto", "src/google/protobuf/unittest_import_public.proto", ], - outs = [ - "TestAllTypesProto3Kt.kt", - "TestEmptyMessageProto3Kt.kt", - ], + strip_import_prefix = "src", +) + +internal_gen_kt_protos( + name = "gen_kotlin_proto3_unittest_lite", + deps = [":kt_proto3_unittest"], + lite = True, visibility = ["//java:__subpackages__"], - cmd = "$(location //:protoc) " + - "--kotlin_out=shared,immutable:$(@D) -Isrc/ " + - "$(location src/google/protobuf/unittest_proto3.proto) && " + - "cp $(@D)/proto3_unittest/TestAllTypesKt.kt " + - "$(location TestAllTypesProto3Kt.kt) && " + - "cp $(@D)/proto3_unittest/TestEmptyMessageKt.kt " + - "$(location TestEmptyMessageProto3Kt.kt)", - tools = ["//:protoc"], ) +internal_gen_kt_protos( + name = "gen_kotlin_proto3_unittest", + deps = [":kt_proto3_unittest"], + visibility = ["//java:__subpackages__"], +) diff --git a/CHANGES.txt b/CHANGES.txt index 6a1e8b91b2..6491e2bf23 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -21,6 +21,7 @@ Unreleased Changes (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) * Report original exceptions when parsing JSON * Add more info to @deprecated javadoc for set/get/has methods * Fix initialization bug in doc comment line numbers + * Fix comments for message set wire format. Kotlin * Add orNull extensions for optional message fields in Kotlin. @@ -46,6 +47,8 @@ Unreleased Changes (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) zone. If omitted or None, the function returns a timezone-naive UTC datetime (as previously). * Adds client_streaming and server_streaming fields to MethodDescriptor. + * Add "ensure_ascii" parameter to json_format.MessageToJson. This allows smaller + JSON serializations with UTF-8 or other non-ASCII encodings. Compiler * Migrate IsDefault(const std::string*) and UnsafeSetDefault(const std::string*) @@ -63,6 +66,7 @@ Unreleased Changes (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) * Do not log/report the same descriptor symbol multiple times if it contains more than one invalid character. * Add UnknownFieldSet::SerializeToString and SerializeToCodedStream. + * Remove explicit default pointers and deprecated API from protocol compiler Arenas * Change Repeated*Field to reuse memory when using arenas. @@ -80,6 +84,10 @@ Unreleased Changes (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) * Generate narrower code * Fix https://github.com/protocolbuffers/protobuf/issues/9378 by removing shadowed _cached_size_ field + * Remove GetPointer() and explicit nullptr defaults. + * add proto_h flag for speeding up large builds + * Add missing overload for reference wrapped fields. + 2022-01-28 version 3.19.4 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) diff --git a/Makefile.am b/Makefile.am index 858eedfa74..fd304a91f7 100644 --- a/Makefile.am +++ b/Makefile.am @@ -54,6 +54,7 @@ csharp_EXTRA_DIST= \ csharp/CHANGES.txt \ csharp/Google.Protobuf.Tools.targets \ csharp/Google.Protobuf.Tools.nuspec \ + csharp/NuGet.Config \ csharp/README.md \ csharp/build_packages.bat \ csharp/build_tools.sh \ @@ -181,10 +182,14 @@ csharp_EXTRA_DIST= \ csharp/src/Google.Protobuf/Collections/ProtobufEqualityComparers.cs \ csharp/src/Google.Protobuf/Collections/ReadOnlyDictionary.cs \ csharp/src/Google.Protobuf/Collections/RepeatedField.cs \ + csharp/src/Google.Protobuf/Compatibility/DynamicallyAccessedMembersAttribute.cs \ + csharp/src/Google.Protobuf/Compatibility/DynamicallyAccessedMemberTypes.cs \ csharp/src/Google.Protobuf/Compatibility/MethodInfoExtensions.cs \ csharp/src/Google.Protobuf/Compatibility/PropertyInfoExtensions.cs \ + csharp/src/Google.Protobuf/Compatibility/RequiresUnreferencedCodeAttribute.cs \ csharp/src/Google.Protobuf/Compatibility/StreamExtensions.cs \ csharp/src/Google.Protobuf/Compatibility/TypeExtensions.cs \ + csharp/src/Google.Protobuf/Compatibility/UnconditionalSuppressMessageAttribute.cs \ csharp/src/Google.Protobuf/Extension.cs \ csharp/src/Google.Protobuf/ExtensionRegistry.cs \ csharp/src/Google.Protobuf/ExtensionSet.cs \ @@ -303,6 +308,7 @@ java_EXTRA_DIST= java/core/src/main/java/com/google/protobuf/CodedInputStreamReader.java \ java/core/src/main/java/com/google/protobuf/CodedOutputStream.java \ java/core/src/main/java/com/google/protobuf/CodedOutputStreamWriter.java \ + java/core/src/main/java/com/google/protobuf/CompileTimeConstant.java \ java/core/src/main/java/com/google/protobuf/DescriptorMessageInfoFactory.java \ java/core/src/main/java/com/google/protobuf/Descriptors.java \ java/core/src/main/java/com/google/protobuf/DiscardUnknownFieldsParser.java \ @@ -326,6 +332,7 @@ java_EXTRA_DIST= java/core/src/main/java/com/google/protobuf/GeneratedMessageInfoFactory.java \ java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java \ java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java \ + java/core/src/main/java/com/google/protobuf/InlineMe.java \ java/core/src/main/java/com/google/protobuf/IntArrayList.java \ java/core/src/main/java/com/google/protobuf/Internal.java \ java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java \ @@ -437,9 +444,9 @@ java_EXTRA_DIST= java/core/src/test/java/com/google/protobuf/ExtensionRegistryFactoryTest.java \ java/core/src/test/java/com/google/protobuf/FieldPresenceTest.java \ java/core/src/test/java/com/google/protobuf/FloatArrayListTest.java \ - java/core/src/test/java/com/google/protobuf/ForceFieldBuildersPreRun.java \ java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java \ java/core/src/test/java/com/google/protobuf/IntArrayListTest.java \ + java/core/src/test/java/com/google/protobuf/InvalidProtocolBufferExceptionTest.java \ java/core/src/test/java/com/google/protobuf/IsValidUtf8Test.java \ java/core/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java \ java/core/src/test/java/com/google/protobuf/LazyFieldLiteTest.java \ @@ -464,15 +471,12 @@ java_EXTRA_DIST= java/core/src/test/java/com/google/protobuf/Proto2ExtensionLookupSchemaTest.java \ java/core/src/test/java/com/google/protobuf/Proto2LiteSchemaTest.java \ java/core/src/test/java/com/google/protobuf/Proto2MessageFactory.java \ - java/core/src/test/java/com/google/protobuf/Proto2MessageInfoFactory.java \ java/core/src/test/java/com/google/protobuf/Proto2MessageLiteFactory.java \ java/core/src/test/java/com/google/protobuf/Proto2SchemaTest.java \ java/core/src/test/java/com/google/protobuf/Proto2UnknownEnumValueTest.java \ java/core/src/test/java/com/google/protobuf/Proto3LiteSchemaTest.java \ java/core/src/test/java/com/google/protobuf/Proto3MessageFactory.java \ - java/core/src/test/java/com/google/protobuf/Proto3MessageInfoFactory.java \ java/core/src/test/java/com/google/protobuf/Proto3MessageLiteFactory.java \ - java/core/src/test/java/com/google/protobuf/Proto3MessageLiteInfoFactory.java \ java/core/src/test/java/com/google/protobuf/Proto3SchemaTest.java \ java/core/src/test/java/com/google/protobuf/ProtobufArrayListTest.java \ java/core/src/test/java/com/google/protobuf/RepeatedFieldBuilderV3Test.java \ @@ -493,9 +497,9 @@ java_EXTRA_DIST= java/core/src/test/java/com/google/protobuf/TypeRegistryTest.java \ java/core/src/test/java/com/google/protobuf/UnknownEnumValueTest.java \ java/core/src/test/java/com/google/protobuf/UnknownFieldSetTest.java \ + java/core/src/test/java/com/google/protobuf/UnknownFieldSetPerformanceTest.java \ java/core/src/test/java/com/google/protobuf/UnmodifiableLazyStringListTest.java \ java/core/src/test/java/com/google/protobuf/Utf8Test.java \ - java/core/src/test/java/com/google/protobuf/Utf8Utils.java \ java/core/src/test/java/com/google/protobuf/WellKnownTypesTest.java \ java/core/src/test/java/com/google/protobuf/WireFormatLiteTest.java \ java/core/src/test/java/com/google/protobuf/WireFormatTest.java \ @@ -504,6 +508,7 @@ java_EXTRA_DIST= java/core/src/test/proto/com/google/protobuf/any_test.proto \ java/core/src/test/proto/com/google/protobuf/cached_field_size_test.proto \ java/core/src/test/proto/com/google/protobuf/deprecated_file.proto \ + java/core/src/test/proto/com/google/protobuf/dynamic_message_test.proto \ java/core/src/test/proto/com/google/protobuf/field_presence_test.proto \ java/core/src/test/proto/com/google/protobuf/lazy_fields_lite.proto \ java/core/src/test/proto/com/google/protobuf/lite_equals_and_hash.proto \ @@ -512,7 +517,6 @@ java_EXTRA_DIST= java/core/src/test/proto/com/google/protobuf/map_initialization_order_test.proto \ java/core/src/test/proto/com/google/protobuf/map_lite_test.proto \ java/core/src/test/proto/com/google/protobuf/map_test.proto \ - java/core/src/test/proto/com/google/protobuf/message_lite_extension_util_test.proto\ java/core/src/test/proto/com/google/protobuf/multiple_files_test.proto \ java/core/src/test/proto/com/google/protobuf/nested_builders_test.proto \ java/core/src/test/proto/com/google/protobuf/nested_extension.proto \ @@ -535,19 +539,22 @@ java_EXTRA_DIST= java/core/src/test/proto/com/google/protobuf/wrappers_test.proto \ java/internal/BUILD \ java/internal/testing.bzl \ + java/kotlin/BUILD \ java/kotlin/generate-sources-build.xml \ java/kotlin/generate-test-sources-build.xml \ java/kotlin/pom.xml \ + java/kotlin/pom_template.xml \ + java/kotlin/src/main/kotlin/com/google/protobuf/Anies.kt \ java/kotlin/src/main/kotlin/com/google/protobuf/ByteStrings.kt \ java/kotlin/src/main/kotlin/com/google/protobuf/DslList.kt \ java/kotlin/src/main/kotlin/com/google/protobuf/DslMap.kt \ java/kotlin/src/main/kotlin/com/google/protobuf/DslProxy.kt \ java/kotlin/src/main/kotlin/com/google/protobuf/ExtendableMessageExtensions.kt \ - java/kotlin/src/main/kotlin/com/google/protobuf/ExtendableMessageLiteExtensions.kt\ java/kotlin/src/main/kotlin/com/google/protobuf/ExtensionList.kt \ java/kotlin/src/main/kotlin/com/google/protobuf/OnlyForUseByGeneratedProtoCode.kt\ java/kotlin/src/main/kotlin/com/google/protobuf/ProtoDslMarker.kt \ java/kotlin/src/main/kotlin/com/google/protobuf/UnmodifiableCollections.kt \ + java/kotlin/src/test/kotlin/com/google/protobuf/AniesTest.kt \ java/kotlin/src/test/kotlin/com/google/protobuf/ByteStringsTest.kt \ java/kotlin/src/test/kotlin/com/google/protobuf/DslListTest.kt \ java/kotlin/src/test/kotlin/com/google/protobuf/DslMapTest.kt \ @@ -559,13 +566,20 @@ java_EXTRA_DIST= java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto3.proto \ java/kotlin/src/test/proto/com/google/protobuf/example_extensible_message.proto \ java/kotlin/src/test/proto/com/google/protobuf/multiple_files_proto3.proto \ + java/kotlin-lite/BUILD \ java/kotlin-lite/generate-sources-build.xml \ java/kotlin-lite/generate-test-sources-build.xml \ java/kotlin-lite/lite.awk \ java/kotlin-lite/pom.xml \ + java/kotlin-lite/pom_template.xml \ java/kotlin-lite/process-lite-sources-build.xml \ + java/kotlin-lite/src/main/kotlin/com/google/protobuf/ExtendableMessageLiteExtensions.kt\ java/kotlin-lite/src/test/kotlin/com/google/protobuf/ExtendableMessageLiteExtensionsTest.kt\ + java/kotlin-lite/src/test/kotlin/com/google/protobuf/Proto3LiteTest.kt \ java/kotlin-lite/src/test/kotlin/com/google/protobuf/Proto2LiteTest.kt \ + java/kotlin-lite/src/test/proto/com/google/protobuf/evil_names_proto2.proto \ + java/kotlin-lite/src/test/proto/com/google/protobuf/evil_names_proto3.proto \ + java/kotlin-lite/src/test/proto/com/google/protobuf/multiple_files_proto3.proto \ java/lite.md \ java/lite/BUILD \ java/lite/generate-sources-build.xml \ @@ -575,7 +589,6 @@ java_EXTRA_DIST= java/lite/pom_template.xml \ java/lite/process-lite-sources-build.xml \ java/lite/src/test/java/com/google/protobuf/LiteTest.java \ - java/lite/src/test/java/com/google/protobuf/Proto2MessageLiteInfoFactory.java \ java/BUILD \ java/pom.xml \ java/util/BUILD \ @@ -588,10 +601,12 @@ java_EXTRA_DIST= java/util/src/main/java/com/google/protobuf/util/Structs.java \ java/util/src/main/java/com/google/protobuf/util/Timestamps.java \ java/util/src/main/java/com/google/protobuf/util/Values.java \ + java/util/src/test/java/com/google/protobuf/util/DurationsTest.java \ java/util/src/test/java/com/google/protobuf/util/FieldMaskTreeTest.java \ java/util/src/test/java/com/google/protobuf/util/FieldMaskUtilTest.java \ java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java \ java/util/src/test/java/com/google/protobuf/util/StructsTest.java \ + java/util/src/test/java/com/google/protobuf/util/TimestampsTest.java \ java/util/src/test/java/com/google/protobuf/util/ValuesTest.java \ java/util/src/test/proto/com/google/protobuf/util/json_test.proto @@ -1372,63 +1387,68 @@ js_EXTRA_DIST= \ all_EXTRA_DIST=$(csharp_EXTRA_DIST) $(java_EXTRA_DIST) $(objectivec_EXTRA_DIST) $(php_EXTRA_DIST) $(python_EXTRA_DIST) $(ruby_EXTRA_DIST) $(js_EXTRA_DIST) -EXTRA_DIST = $(@DIST_LANG@_EXTRA_DIST) \ - autogen.sh \ - generate_descriptor_proto.sh \ - README.md \ - LICENSE \ - CONTRIBUTORS.txt \ - CHANGES.txt \ - update_file_lists.sh \ - BUILD \ - WORKSPACE \ - cmake/CMakeLists.txt \ - cmake/README.md \ - cmake/conformance.cmake \ - cmake/examples.cmake \ - cmake/extract_includes.bat.in \ - cmake/install.cmake \ - cmake/libprotobuf.cmake \ - cmake/libprotobuf-lite.cmake \ - cmake/libprotoc.cmake \ - cmake/protobuf-config-version.cmake.in \ - cmake/protobuf-config.cmake.in \ - cmake/protobuf-lite.pc.cmake \ - cmake/protobuf-module.cmake.in \ - cmake/protobuf-options.cmake \ - cmake/protobuf.pc.cmake \ - cmake/protoc.cmake \ - cmake/tests.cmake \ - cmake/version.rc.in \ - compiler_config_setting.bzl \ - build_files_updated_unittest.sh \ - cc_proto_blacklist_test.bzl \ - editors/README.txt \ - editors/proto.vim \ - editors/protobuf-mode.el \ - examples/AddPerson.java \ - examples/BUILD \ - examples/CMakeLists.txt \ - examples/ListPeople.java \ - examples/Makefile \ - examples/README.md \ - examples/WORKSPACE \ - examples/add_person.cc \ - examples/add_person.dart \ - examples/add_person.go \ - examples/add_person.py \ - examples/add_person_test.go \ - examples/addressbook.proto \ - examples/list_people.cc \ - examples/list_people.dart \ - examples/list_people.go \ - examples/list_people.py \ - examples/list_people_test.go \ - examples/pubspec.yaml \ - protobuf.bzl \ - protobuf_deps.bzl \ - third_party/zlib.BUILD \ - util/python/BUILD \ +EXTRA_DIST = $(@DIST_LANG@_EXTRA_DIST) \ + autogen.sh \ + generate_descriptor_proto.sh \ + README.md \ + LICENSE \ + CONTRIBUTORS.txt \ + CHANGES.txt \ + update_file_lists.sh \ + BUILD \ + WORKSPACE \ + cmake/CMakeLists.txt \ + cmake/README.md \ + cmake/conformance.cmake \ + cmake/examples.cmake \ + cmake/extract_includes.bat.in \ + cmake/install.cmake \ + cmake/libprotobuf.cmake \ + cmake/libprotobuf-lite.cmake \ + cmake/libprotoc.cmake \ + cmake/protobuf-config-version.cmake.in \ + cmake/protobuf-config.cmake.in \ + cmake/protobuf-lite.pc.cmake \ + cmake/protobuf-module.cmake.in \ + cmake/protobuf-options.cmake \ + cmake/protobuf.pc.cmake \ + cmake/protoc.cmake \ + cmake/tests.cmake \ + cmake/version.rc.in \ + compiler_config_setting.bzl \ + build_files_updated_unittest.sh \ + cc_proto_blacklist_test.bzl \ + editors/README.txt \ + editors/proto.vim \ + editors/protobuf-mode.el \ + examples/AddPerson.java \ + examples/BUILD \ + examples/CMakeLists.txt \ + examples/ListPeople.java \ + examples/Makefile \ + examples/README.md \ + examples/WORKSPACE \ + examples/add_person.cc \ + examples/add_person.dart \ + examples/add_person.py \ + examples/addressbook.proto \ + examples/go/cmd/add_person/add_person.go \ + examples/go/cmd/add_person/add_person_test.go \ + examples/go/cmd/list_people/list_people.go \ + examples/go/cmd/list_people/list_people_test.go \ + examples/go/go.sum \ + examples/go/go.mod \ + examples/list_people.cc \ + examples/list_people.dart \ + examples/list_people.py \ + examples/pubspec.yaml \ + maven_install.json \ + protobuf.bzl \ + protobuf_deps.bzl \ + protobuf_release.bzl \ + protobuf_version.bzl \ + third_party/zlib.BUILD \ + util/python/BUILD \ internal.bzl diff --git a/Protobuf-C++.podspec b/Protobuf-C++.podspec index 9b8fae51a7..035ae366de 100644 --- a/Protobuf-C++.podspec +++ b/Protobuf-C++.podspec @@ -1,9 +1,9 @@ Pod::Spec.new do |s| s.name = 'Protobuf-C++' - s.version = '3.18.1' + s.version = '3.19.4' s.summary = 'Protocol Buffers v3 runtime library for C++.' s.homepage = 'https://github.com/google/protobuf' - s.license = '3-Clause BSD License' + s.license = 'BSD-3-Clause' s.authors = { 'The Protocol Buffers contributors' => 'protobuf@googlegroups.com' } s.cocoapods_version = '>= 1.0' diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am index 2b2204d082..3ab35e37ca 100644 --- a/benchmarks/Makefile.am +++ b/benchmarks/Makefile.am @@ -165,7 +165,7 @@ python_add_init: protoc_middleman protoc_middleman2 done \ done -python_cpp_pkg_flags = `pkg-config --cflags --libs python` +python_cpp_pkg_flags = `pkg-config --cflags --libs python3` lib_LTLIBRARIES = libbenchmark_messages.la libbenchmark_messages_la_SOURCES = python/python_benchmark_messages.cc @@ -186,7 +186,7 @@ python-pure-python-benchmark: python_add_init @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/.libs >> python-pure-python-benchmark @echo export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=\'python\' >> python-pure-python-benchmark @echo cp $(srcdir)/python/py_benchmark.py tmp >> python-pure-python-benchmark - @echo python tmp/py_benchmark.py '$$@' >> python-pure-python-benchmark + @echo python3 tmp/py_benchmark.py '$$@' >> python-pure-python-benchmark @chmod +x python-pure-python-benchmark python-cpp-reflection-benchmark: python_add_init @@ -196,7 +196,7 @@ python-cpp-reflection-benchmark: python_add_init @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/.libs >> python-cpp-reflection-benchmark @echo export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=\'cpp\' >> python-cpp-reflection-benchmark @echo cp $(srcdir)/python/py_benchmark.py tmp >> python-cpp-reflection-benchmark - @echo python tmp/py_benchmark.py '$$@' >> python-cpp-reflection-benchmark + @echo python3 tmp/py_benchmark.py '$$@' >> python-cpp-reflection-benchmark @chmod +x python-cpp-reflection-benchmark python-cpp-generated-code-benchmark: python_add_init libbenchmark_messages.la @@ -206,7 +206,7 @@ python-cpp-generated-code-benchmark: python_add_init libbenchmark_messages.la @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/.libs >> python-cpp-generated-code-benchmark @echo export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=\'cpp\' >> python-cpp-generated-code-benchmark @echo cp $(srcdir)/python/py_benchmark.py tmp >> python-cpp-generated-code-benchmark - @echo python tmp/py_benchmark.py --cpp_generated '$$@' >> python-cpp-generated-code-benchmark + @echo python3 tmp/py_benchmark.py --cpp_generated '$$@' >> python-cpp-generated-code-benchmark @chmod +x python-cpp-generated-code-benchmark python-pure-python: python-pure-python-benchmark diff --git a/benchmarks/README.md b/benchmarks/README.md index f708afd9cb..13a8843d2e 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -87,7 +87,17 @@ To run all the benchmark dataset: ### Java: +First build the Java binary in the usual way with Maven: + +``` +$ cd java +$ mvn install +``` + +Assuming that completes successfully, + ``` +$ cd ../benchmarks $ make java ``` diff --git a/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java index a4402481b0..97c7376cd9 100644 --- a/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java +++ b/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java @@ -1,38 +1,27 @@ - package com.google.protobuf; import com.google.caliper.BeforeExperiment; -import com.google.caliper.AfterExperiment; import com.google.caliper.Benchmark; import com.google.caliper.Param; -import com.google.caliper.api.VmOptions; -import com.google.protobuf.ByteString; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; -import com.google.protobuf.Message; import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.FileWriter; import java.io.IOException; import java.io.RandomAccessFile; import java.util.ArrayList; import java.util.List; -// Caliper set CICompilerCount to 1 for making sure compilation doesn't run in parallel with itself, -// This makes TieredCompilation not working. We just disable TieredCompilation by default. In master -// branch this has been disabled by default in caliper: -// https://github.com/google/caliper/blob/master/caliper-runner/src/main/java/com/google/caliper/runner/target/Jvm.java#L38:14 -// But this haven't been added into most recent release. -@VmOptions("-XX:-TieredCompilation") +/** + * Basic benchmarks for Java protobuf parsing. + */ public class ProtoCaliperBenchmark { public enum BenchmarkMessageType { GOOGLE_MESSAGE1_PROTO3 { - @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } + @Override + ExtensionRegistry getExtensionRegistry() { + return ExtensionRegistry.newInstance(); + } @Override Message getDefaultInstance() { return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1 @@ -40,7 +29,9 @@ public class ProtoCaliperBenchmark { } }, GOOGLE_MESSAGE1_PROTO2 { - @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } + @Override ExtensionRegistry getExtensionRegistry() { + return ExtensionRegistry.newInstance(); + } @Override Message getDefaultInstance() { return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1 @@ -48,7 +39,10 @@ public class ProtoCaliperBenchmark { } }, GOOGLE_MESSAGE2 { - @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } + @Override + ExtensionRegistry getExtensionRegistry() { + return ExtensionRegistry.newInstance(); + } @Override Message getDefaultInstance() { return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance(); diff --git a/benchmarks/python/python_benchmark_messages.cc b/benchmarks/python/python_benchmark_messages.cc index ded16fe96e..f6ddcf3185 100644 --- a/benchmarks/python/python_benchmark_messages.cc +++ b/benchmarks/python/python_benchmark_messages.cc @@ -7,13 +7,18 @@ #include "datasets/google_message3/benchmark_message3.pb.h" #include "datasets/google_message4/benchmark_message4.pb.h" -static PyMethodDef python_benchmark_methods[] = { - {NULL, NULL, 0, NULL} /* Sentinel */ -}; +static struct PyModuleDef _module = {PyModuleDef_HEAD_INIT, + "libbenchmark_messages", + "Benchmark messages Python module", + -1, + NULL, + NULL, + NULL, + NULL, + NULL}; - -PyMODINIT_FUNC -initlibbenchmark_messages() { +extern "C" { +PyMODINIT_FUNC PyInit_libbenchmark_messages() { benchmarks::BenchmarkDataset().descriptor(); benchmarks::proto3::GoogleMessage1().descriptor(); benchmarks::proto2::GoogleMessage1().descriptor(); @@ -21,9 +26,6 @@ initlibbenchmark_messages() { benchmarks::google_message3::GoogleMessage3().descriptor(); benchmarks::google_message4::GoogleMessage4().descriptor(); - PyObject *m; - - m = Py_InitModule("libbenchmark_messages", python_benchmark_methods); - if (m == NULL) - return; + return PyModule_Create(&_module); +} } diff --git a/benchmarks/util/result_parser.py b/benchmarks/util/result_parser.py index bdf3a9980a..d3251a8b26 100644 --- a/benchmarks/util/result_parser.py +++ b/benchmarks/util/result_parser.py @@ -61,7 +61,7 @@ def __parse_cpp_result(filename): return if filename[0] != '/': filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename - with open(filename, "rb") as f: + with open(filename, encoding="utf-8") as f: results = json.loads(f.read()) for benchmark in results["benchmarks"]: data_filename = "".join( @@ -96,7 +96,7 @@ def __parse_synthetic_result(filename): return if filename[0] != "/": filename = os.path.dirname(os.path.abspath(__file__)) + "/" + filename - with open(filename, "rb") as f: + with open(filename, encoding="utf-8") as f: results = json.loads(f.read()) for benchmark in results["benchmarks"]: __results.append({ @@ -126,7 +126,7 @@ def __parse_python_result(filename): return if filename[0] != '/': filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename - with open(filename, "rb") as f: + with open(filename, encoding="utf-8") as f: results_list = json.loads(f.read()) for results in results_list: for result in results: @@ -176,7 +176,7 @@ def __parse_java_result(filename): return if filename[0] != '/': filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename - with open(filename, "rb") as f: + with open(filename, encoding="utf-8") as f: results = json.loads(f.read()) for result in results: total_weight = 0 @@ -212,7 +212,7 @@ def __parse_go_result(filename): return if filename[0] != '/': filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename - with open(filename, "rb") as f: + with open(filename, encoding="utf-8") as f: for line in f: result_list = re.split(r"[\ \t]+", line) if result_list[0][:9] != "Benchmark": @@ -252,7 +252,7 @@ def __parse_custom_result(filename, language): return if filename[0] != '/': filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename - with open(filename, "rb") as f: + with open(filename, encoding="utf-8") as f: results = json.loads(f.read()) for result in results: _, avg_size = __get_data_size(result["filename"]) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 51e8478f6e..2366d4b427 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -41,6 +41,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES Intel) endif() # Options +option(protobuf_INSTALL "Install protobuf binaries and files" ON) if(WITH_PROTOC) set(protobuf_PROTOC_EXE ${WITH_PROTOC} CACHE FILEPATH "Protocol Buffer Compiler executable" FORCE) endif() @@ -182,7 +183,11 @@ else (protobuf_BUILD_SHARED_LIBS) # making programmatic control difficult. Prefer the functionality in newer # CMake versions when available. if(CMAKE_VERSION VERSION_GREATER 3.15 OR CMAKE_VERSION VERSION_EQUAL 3.15) - set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreaded$<$:Debug>) + if (protobuf_MSVC_STATIC_RUNTIME) + set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreaded$<$:Debug>) + else() + set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreaded$<$:Debug>DLL) + endif() else() # In case we are building static libraries, link also the runtime library statically # so that MSVCR*.DLL is not required at runtime. @@ -210,9 +215,7 @@ if (MSVC) add_definitions(/utf-8) # MSVC warning suppressions add_definitions( - /wd4018 # 'expression' : signed/unsigned mismatch /wd4065 # switch statement contains 'default' but no 'case' labels - /wd4146 # unary minus operator applied to unsigned type, result still unsigned /wd4244 # 'conversion' conversion from 'type1' to 'type2', possible loss of data /wd4251 # 'identifier' : class 'type' needs to have dll-interface to be used by clients of class 'type2' /wd4267 # 'var' : conversion from 'size_t' to 'type', possible loss of data @@ -294,6 +297,7 @@ if (protobuf_BUILD_TESTS OR protobuf_BUILD_CONFORMANCE OR protobuf_BUILD_EXAMPLE endif () if (protobuf_BUILD_TESTS) + enable_testing() include(tests.cmake) endif (protobuf_BUILD_TESTS) @@ -301,7 +305,9 @@ if (protobuf_BUILD_CONFORMANCE) include(conformance.cmake) endif (protobuf_BUILD_CONFORMANCE) -include(install.cmake) +if (protobuf_INSTALL) + include(install.cmake) +endif (protobuf_INSTALL) if (protobuf_BUILD_EXAMPLES) include(examples.cmake) diff --git a/cmake/README.md b/cmake/README.md index 89d00c1b6f..3fee4a0414 100644 --- a/cmake/README.md +++ b/cmake/README.md @@ -36,6 +36,10 @@ If *git* command is not available from *Command Prompt*, add it to system *PATH* C:\Path\to>set PATH=%PATH%;C:\Program Files\Git\cmd +Optionally, you will want to download [ninja](https://ninja-build.org/) and add it to your *PATH* variable. + + C:\Path\to>set PATH=%PATH%;C:\tools\ninja + Good. Now you are ready to continue. Getting Sources @@ -52,29 +56,25 @@ download `protobuf-all-[VERSION].tar.gz`. Or you can use git to clone from protobuf git repository. - C:\Path\to> git clone -b [release_tag] https://github.com/protocolbuffers/protobuf.git + C:\Path\to> mkdir src & cd src + C:\Path\to\src> git clone -b [release_tag] https://github.com/protocolbuffers/protobuf.git Where *[release_tag]* is a git tag like *v3.0.0-beta-1* or a branch name like *master* if you want to get the latest code. Go to the project folder: - C:\Path\to>cd protobuf - C:\Path\to\protobuf> + C:\Path\to\src> cd protobuf + C:\Path\to\src\protobuf> Remember to update any submodules if you are using git clone (you can skip this step if you are using a release .tar.gz or .zip package): ```console -C:\Path\to> git submodule update --init --recursive +C:\Path\to\src\protobuf> git submodule update --init --recursive ``` -Now go to *cmake* folder in protobuf sources: - - C:\Path\to\protobuf>cd cmake - C:\Path\to\protobuf\cmake> - -Good. Now you are ready to *CMake* configuration. +Good. Now you are ready for *CMake* configuration. CMake Configuration =================== @@ -82,71 +82,119 @@ CMake Configuration *CMake* supports a lot of different [generators](http://www.cmake.org/cmake/help/latest/manual/cmake-generators.7.html) for various native build systems. -We are only interested in -[Makefile](http://www.cmake.org/cmake/help/latest/manual/cmake-generators.7.html#makefile-generators) -and -[Visual Studio](http://www.cmake.org/cmake/help/latest/manual/cmake-generators.7.html#visual-studio-generators) -generators. -We will use shadow building to separate the temporary files from the protobuf source code. +Of most interest to Windows programmers are the following: + +* [Makefile](http://www.cmake.org/cmake/help/latest/manual/cmake-generators.7.html#makefile-generators). + This generates NMake Makefiles for Visual Studio. These work, but they are rather slow. + +* [Visual Studio](http://www.cmake.org/cmake/help/latest/manual/cmake-generators.7.html#visual-studio-generators) + This generates a Visual Studio solution for the project. + +* [Ninja](https://cmake.org/cmake/help/latest/manual/cmake-generators.7.html#ninja-generator) + This uses the external tool [Ninja](https://ninja-build.org/) to build. It is the fastest solution available. + +Note that as of Visual Studio 2015, Visual Studio includes +[support for opening directly CMake-based projects](https://docs.microsoft.com/en-us/cpp/build/cmake-projects-in-visual-studio). + +It is considered good practice not to build CMake projects in the source tree but in a separate folder. Create a temporary *build* folder and change your working directory to it: - C:\Path\to\protobuf\cmake>mkdir build & cd build - C:\Path\to\protobuf\cmake\build> + mkdir C:\Path\to\build\protobuf + cd C:\Path\to\build\protobuf + C:\Path\to\build\protobuf> -The *Makefile* generator can build the project in only one configuration, so you need to build +The *Makefile* and *Ninja* generators can build the project in only one configuration, so you need to build a separate folder for each configuration. -To start using a *Release* configuration: +To start using a *Release* configuration via the *NMmake* generator: - C:\Path\to\protobuf\cmake\build>mkdir release & cd release - C:\Path\to\protobuf\cmake\build\release>cmake -G "NMake Makefiles" ^ + C:\Path\to\build\protobuf>mkdir release & cd release + C:\Path\to\build\protobuf\release>cmake -G "NMake Makefiles" ^ -DCMAKE_BUILD_TYPE=Release ^ - -DCMAKE_INSTALL_PREFIX=../../../../install ^ - ../.. + -DCMAKE_INSTALL_PREFIX=C:\Path\to\install ^ + C:\Path\to\src\protobuf -It will generate *nmake* *Makefile* in current directory. +It will generate a *NMake* *Makefile* in the current directory. -To use *Debug* configuration: +To use *Debug* configuration using *Ninja*: - C:\Path\to\protobuf\cmake\build>mkdir debug & cd debug - C:\Path\to\protobuf\cmake\build\debug>cmake -G "NMake Makefiles" ^ + C:\Path\to\build\protobuf>mkdir debug & cd debug + C:\Path\to\build\protobuf\debug>cmake -G "Ninja" ^ -DCMAKE_BUILD_TYPE=Debug ^ - -DCMAKE_INSTALL_PREFIX=../../../../install ^ - ../.. + -DCMAKE_INSTALL_PREFIX=C:\Path\to\install ^ + C:\Path\to\src\protobuf -It will generate *nmake* *Makefile* in current directory. +It will generate *Ninja* build scripts in current directory. -To create *Visual Studio* solution file: +The *Visual Studio* generator is multi-configuration: it will generate a single *.sln* file that can be used for both *Debug* and *Release*: - C:\Path\to\protobuf\cmake\build>mkdir solution & cd solution - C:\Path\to\protobuf\cmake\build\solution>cmake -G "Visual Studio 16 2019" ^ - -DCMAKE_INSTALL_PREFIX=../../../../install ^ - ../.. + C:\Path\to\build\protobuf>mkdir solution & cd solution + C:\Path\to\build\protobuf\solution>cmake -G "Visual Studio 16 2019" ^ + -DCMAKE_INSTALL_PREFIX=C:\Path\to\install ^ + C:\Path\to\src\protobuf It will generate *Visual Studio* solution file *protobuf.sln* in current directory. -If the *gmock* directory does not exist, and you do not want to build protobuf unit tests, -you need to add *cmake* command argument `-Dprotobuf_BUILD_TESTS=OFF` to disable testing. +Unit Tests +---------- + +Unit tests are being built along with the rest of protobuf. The unit tests require Google Mock (now a part of Google Test). + +A copy of [Google Test](https://github.com/google/googletest) is included as a Git submodule in the `third-party/googletest` folder. +(You do need to initialize the Git submodules as explained above.) + +Alternately, you may want to use protobuf in a larger set-up, you may want to use that standard CMake approach where +you build and install a shared copy of Google Test. + +After you've built and installed your Google Test copy, you need add the following definition to your *cmake* command line +during the configuration step: `-Dprotobuf_USE_EXTERNAL_GTEST=ON`. +This will cause the standard CMake `find_package(GTest REQUIRED)` to be used. -To make a *Visual Studio* file for Visual Studio 16 2019, create the *Visual Studio* -solution file above and edit the CMakeCache file. +[find_package](https://cmake.org/cmake/help/latest/command/find_package.html) will search in a default location, +which on Windows is *C:\Program Files*. This is most likely not what you want. You will want instead to search for +Google Test in your project's root directory (i.e. the same directory you've passed to `CMAKE_INSTALL_PREFIX` when +building Google Test). For this, you need to set the `CMAKE_PREFIX_PATH` CMake variable. (There are other ways in CMake, +see the [manual](https://cmake.org/cmake/help/latest/command/find_package.html) for details.) - C:Path\to\protobuf\cmake\build\solution\CMakeCache +For example: -Then create the *Visual Studio* solution file again + C:\Path\to\build\protobuf>mkdir solution & cd solution + C:\Path\to\build\protobuf\solution>cmake -G "Visual Studio 16 2019" ^ + -DCMAKE_INSTALL_PREFIX=C:\Path\to\install ^ + -DCMAKE_PREFIX_PATH=C:\Path\to\my_big_project ^ + -Dprotobuf_USE_EXTERNAL_GTEST=ON ^ + C:\Path\to\src\protobuf + +In most cases, `CMAKE_PREFIX_PATH` and `CMAKE_INSTALL_PREFIX` will point to the same directory. + +To disable testing completely, you need to add the following argument to you *cmake* command line: `-Dprotobuf_BUILD_TESTS=OFF`. + +For example: + + C:\Path\to\build\protobuf\solution>cmake -G "Visual Studio 16 2019" ^ + -DCMAKE_INSTALL_PREFIX=C:\Path\to\install ^ + -Dprotobuf_BUILD_TESTS=OFF ^ + C:\Path\to\src\protobuf Compiling ========= -To compile protobuf: +The standard way to compile a *CMake* project is `cmake --build `. + + +Note that if your generator supports multiple configurations, you will probably want to specify which one to build: - C:\Path\to\protobuf\cmake\build\release>nmake + cmake --build C:\Path\to\build\protobuf\solution --config Release + +You can also run directly the build tool you've configured: + + C:\Path\to\build\protobuf\release>nmake or - C:\Path\to\protobuf\cmake\build\debug>nmake + C:\Path\to\build\protobuf\debug>ninja And wait for the compilation to finish. @@ -164,11 +212,15 @@ Testing To run unit-tests, first you must compile protobuf as described above. Then run: - C:\Path\to\protobuf\cmake\build\release>nmake check + C:\Path\to\protobuf\cmake\build\release>ctest --progress --output-on-failure + +You can also build the `check` target (not idiomatic CMake usage, though): + + C:\Path\to\protobuf\cmake\build\release>cmake --build . --target check or - C:\Path\to\protobuf\cmake\build\debug>nmake check + C:\Path\to\build\protobuf\release>ninja check You can also build project *check* from Visual Studio solution. Yes, it may sound strange, but it works. @@ -183,9 +235,9 @@ You should see output similar to: [==========] 1546 tests from 165 test cases ran. (2529 ms total) [ PASSED ] 1546 tests. -To run specific tests: +To run specific tests, you need to pass some command line arguments to the test program itself: - C:\Path\to\protobuf>cmake\build\release\tests.exe --gtest_filter=AnyTest* + C:\Path\to\build\protobuf\release>tests.exe --gtest_filter=AnyTest* Running main() from gmock_main.cc Note: Google Test filter = AnyTest* [==========] Running 3 tests from 1 test case. @@ -210,13 +262,17 @@ If all tests are passed, safely continue. Installing ========== -To install protobuf to the specified *install* folder: +To install protobuf to the *install* folder you've specified in the configuration step, you need to build the `install` target: + + cmake --build C:\Path\to\build\protobuf\solution --config Release --target install + +Or if you prefer: - C:\Path\to\protobuf\cmake\build\release>nmake install + C:\Path\to\build\protobuf\release>nmake install or - C:\Path\to\protobuf\cmake\build\debug>nmake install + C:\Path\to\build\protobuf\debug>ninja install You can also build project *INSTALL* from Visual Studio solution. It sounds not so strange and it works. @@ -280,16 +336,16 @@ You can also compile it from source by yourself. Getting sources: - C:\Path\to>git clone -b v1.2.8 https://github.com/madler/zlib.git - C:\Path\to>cd zlib + C:\Path\to\src>git clone -b v1.2.8 https://github.com/madler/zlib.git + C:\Path\to\src>cd zlib Compiling and Installing: - C:\Path\to\zlib>mkdir build & cd build - C:\Path\to\zlib\build>mkdir release & cd release - C:\Path\to\zlib\build\release>cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release ^ - -DCMAKE_INSTALL_PREFIX=../../../install ../.. - C:\Path\to\zlib\build\release>nmake & nmake install + C:\Path\to\src\zlib>mkdir C:\Path\to\build\zlib & cd C:\Path\to\build\zlib + C:\Path\to\build\zlib>mkdir release & cd release + C:\Path\to\build\zlib\release>cmake -G "Ninja" -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_INSTALL_PREFIX=C:\Path\to\install C:\Path\to\src\zlib + C:\Path\to\src\zlib\build\release>cmake --build . --target install You can make *debug* version or use *Visual Studio* generator also as before for the protobuf project. @@ -308,8 +364,8 @@ the headers or the .lib file in the right directory. If you already have ZLIB library and headers at some other location on your system then alternatively you can define following configuration flags to locate them: - -DZLIB_INCLUDE_DIR= - -DZLIB_LIB= + -DZLIB_INCLUDE_DIR= + -DZLIB_LIB= Build and testing protobuf as usual. @@ -320,8 +376,6 @@ The following warnings have been disabled while building the protobuf libraries and compiler. You may have to disable some of them in your own project as well, or live with them. -* C4018 - 'expression' : signed/unsigned mismatch -* C4146 - unary minus operator applied to unsigned type, result still unsigned * C4244 - Conversion from 'type1' to 'type2', possible loss of data. * C4251 - 'identifier' : class 'type' needs to have dll-interface to be used by clients of class 'type2' diff --git a/cmake/extract_includes.bat.in b/cmake/extract_includes.bat.in index 2f53be0dad..5169a02356 100644 --- a/cmake/extract_includes.bat.in +++ b/cmake/extract_includes.bat.in @@ -59,11 +59,8 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_enum_reflec copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_enum_util.h" include\google\protobuf\generated_enum_util.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_bases.h" include\google\protobuf\generated_message_bases.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_reflection.h" include\google\protobuf\generated_message_reflection.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_table_driven.h" include\google\protobuf\generated_message_table_driven.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_table_driven_lite.h" include\google\protobuf\generated_message_table_driven_lite.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_tctable_decl.h" include\google\protobuf\generated_message_tctable_decl.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_tctable_impl.h" include\google\protobuf\generated_message_tctable_impl.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_tctable_impl.inc" include\google\protobuf\generated_message_tctable_impl.inc copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_util.h" include\google\protobuf\generated_message_util.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\has_bits.h" include\google\protobuf\has_bits.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\implicit_weak_message.h" include\google\protobuf\implicit_weak_message.h diff --git a/cmake/libprotobuf-lite.cmake b/cmake/libprotobuf-lite.cmake index 0ee65e20f8..5e98748f60 100644 --- a/cmake/libprotobuf-lite.cmake +++ b/cmake/libprotobuf-lite.cmake @@ -101,7 +101,7 @@ if(${CMAKE_SYSTEM_NAME} STREQUAL "Android") target_link_libraries(libprotobuf-lite log) endif() target_include_directories(libprotobuf-lite PUBLIC ${protobuf_source_dir}/src) -if(MSVC AND protobuf_BUILD_SHARED_LIBS) +if(protobuf_BUILD_SHARED_LIBS) target_compile_definitions(libprotobuf-lite PUBLIC PROTOBUF_USE_DLLS PRIVATE LIBPROTOBUF_EXPORTS) diff --git a/cmake/libprotobuf.cmake b/cmake/libprotobuf.cmake index 1f5ae5ab04..ef5a3cf593 100644 --- a/cmake/libprotobuf.cmake +++ b/cmake/libprotobuf.cmake @@ -118,13 +118,15 @@ if(${CMAKE_SYSTEM_NAME} STREQUAL "Android") target_link_libraries(libprotobuf log) endif() target_include_directories(libprotobuf PUBLIC ${protobuf_source_dir}/src) -if(MSVC AND protobuf_BUILD_SHARED_LIBS) +if(protobuf_BUILD_SHARED_LIBS) target_compile_definitions(libprotobuf PUBLIC PROTOBUF_USE_DLLS PRIVATE LIBPROTOBUF_EXPORTS) endif() set_target_properties(libprotobuf PROPERTIES VERSION ${protobuf_VERSION} + # Use only the first SO version component for compatibility with Makefile emitted SONAME. + SOVERSION 30 OUTPUT_NAME ${LIB_PREFIX}protobuf DEBUG_POSTFIX "${protobuf_DEBUG_POSTFIX}") add_library(protobuf::libprotobuf ALIAS libprotobuf) diff --git a/cmake/protobuf-config.cmake.in b/cmake/protobuf-config.cmake.in index 9197625dc2..b4332099c9 100644 --- a/cmake/protobuf-config.cmake.in +++ b/cmake/protobuf-config.cmake.in @@ -11,7 +11,7 @@ function(protobuf_generate) include(CMakeParseArguments) set(_options APPEND_PATH) - set(_singleargs LANGUAGE OUT_VAR EXPORT_MACRO PROTOC_OUT_DIR PLUGIN) + set(_singleargs LANGUAGE OUT_VAR EXPORT_MACRO PROTOC_OUT_DIR PLUGIN PLUGIN_OPTIONS) if(COMMAND target_sources) list(APPEND _singleargs TARGET) endif() @@ -39,9 +39,18 @@ function(protobuf_generate) endif() if(protobuf_generate_EXPORT_MACRO AND protobuf_generate_LANGUAGE STREQUAL cpp) - set(_dll_export_decl "dllexport_decl=${protobuf_generate_EXPORT_MACRO}:") + set(_dll_export_decl "dllexport_decl=${protobuf_generate_EXPORT_MACRO}") endif() - + + foreach(_option ${_dll_export_decl} ${protobuf_generate_PLUGIN_OPTIONS}) + # append comma - not using CMake lists and string replacement as users + # might have semicolons in options + if(_plugin_options) + set( _plugin_options "${_plugin_options},") + endif() + set(_plugin_options "${_plugin_options}${_option}") + endforeach() + if(protobuf_generate_PLUGIN) set(_plugin "--plugin=${protobuf_generate_PLUGIN}") endif() @@ -127,12 +136,20 @@ function(protobuf_generate) endforeach() list(APPEND _generated_srcs_all ${_generated_srcs}) + set(_comment "Running ${protobuf_generate_LANGUAGE} protocol buffer compiler on ${_proto}") + if(protobuf_generate_PROTOC_OPTIONS) + set(_comment "${_comment}, protoc-options: ${protobuf_generate_PROTOC_OPTIONS}") + endif() + if(_plugin_options) + set(_comment "${_comment}, plugin-options: ${_plugin_options}") + endif() + add_custom_command( OUTPUT ${_generated_srcs} - COMMAND protobuf::protoc - ARGS ${protobuf_generate_PROTOC_OPTIONS} --${protobuf_generate_LANGUAGE}_out ${_dll_export_decl}${protobuf_generate_PROTOC_OUT_DIR} ${_plugin} ${_protobuf_include_path} ${_abs_file} + COMMAND protobuf::protoc + ARGS ${protobuf_generate_PROTOC_OPTIONS} --${protobuf_generate_LANGUAGE}_out ${_plugin_options}:${protobuf_generate_PROTOC_OUT_DIR} ${_plugin} ${_protobuf_include_path} ${_abs_file} DEPENDS ${_abs_file} protobuf::protoc - COMMENT "Running ${protobuf_generate_LANGUAGE} protocol buffer compiler on ${_proto}. Custom options: ${protobuf_generate_PROTOC_OPTIONS}" + COMMENT ${_comment} VERBATIM ) endforeach() diff --git a/cmake/protobuf-module.cmake.in b/cmake/protobuf-module.cmake.in index 09b9d29c22..0bb05e38fa 100644 --- a/cmake/protobuf-module.cmake.in +++ b/cmake/protobuf-module.cmake.in @@ -94,7 +94,7 @@ function(_protobuf_find_libraries name filename) elseif(${name}_LIBRARY) # Honor cache entry used by CMake 3.5 and lower. set(${name}_LIBRARIES "${${name}_LIBRARY}" PARENT_SCOPE) - else() + elseif(TARGET protobuf::lib${filename}) get_target_property(${name}_LIBRARY_RELEASE protobuf::lib${filename} LOCATION_RELEASE) get_target_property(${name}_LIBRARY_RELWITHDEBINFO protobuf::lib${filename} @@ -134,23 +134,25 @@ get_target_property(Protobuf_INCLUDE_DIRS protobuf::libprotobuf INTERFACE_INCLUDE_DIRECTORIES) # Set the protoc Executable -get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc - IMPORTED_LOCATION_RELEASE) -if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}") +if(NOT Protobuf_PROTOC_EXECUTABLE AND TARGET protobuf::protoc) get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc - IMPORTED_LOCATION_RELWITHDEBINFO) -endif() -if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}") - get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc - IMPORTED_LOCATION_MINSIZEREL) -endif() -if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}") - get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc - IMPORTED_LOCATION_DEBUG) -endif() -if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}") - get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc - IMPORTED_LOCATION_NOCONFIG) + IMPORTED_LOCATION_RELEASE) + if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}") + get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc + IMPORTED_LOCATION_RELWITHDEBINFO) + endif() + if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}") + get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc + IMPORTED_LOCATION_MINSIZEREL) + endif() + if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}") + get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc + IMPORTED_LOCATION_DEBUG) + endif() + if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}") + get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc + IMPORTED_LOCATION_NOCONFIG) + endif() endif() # Version info variable diff --git a/cmake/tests.cmake b/cmake/tests.cmake index aa069febd2..0b0e1bed8c 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -1,32 +1,41 @@ -if (NOT EXISTS "${PROJECT_SOURCE_DIR}/../third_party/googletest/CMakeLists.txt") - message(FATAL_ERROR - "Cannot find third_party/googletest directory that's needed to " - "build tests. If you use git, make sure you have cloned submodules:\n" - " git submodule update --init --recursive\n" - "If instead you want to skip tests, run cmake with:\n" - " cmake -Dprotobuf_BUILD_TESTS=OFF\n") -endif() +option(protobuf_USE_EXTERNAL_GTEST "Use external Google Test (i.e. not the one in third_party/googletest)" OFF) option(protobuf_ABSOLUTE_TEST_PLUGIN_PATH "Using absolute test_plugin path in tests" ON) mark_as_advanced(protobuf_ABSOLUTE_TEST_PLUGIN_PATH) -set(googlemock_source_dir "${protobuf_source_dir}/third_party/googletest/googlemock") -set(googletest_source_dir "${protobuf_source_dir}/third_party/googletest/googletest") -include_directories( - ${googlemock_source_dir} - ${googletest_source_dir} - ${googletest_source_dir}/include - ${googlemock_source_dir}/include -) +if (protobuf_USE_EXTERNAL_GTEST) + find_package(GTest REQUIRED) +else() + if (NOT EXISTS "${PROJECT_SOURCE_DIR}/../third_party/googletest/CMakeLists.txt") + message(FATAL_ERROR + "Cannot find third_party/googletest directory that's needed to " + "build tests. If you use git, make sure you have cloned submodules:\n" + " git submodule update --init --recursive\n" + "If instead you want to skip tests, run cmake with:\n" + " cmake -Dprotobuf_BUILD_TESTS=OFF\n") + endif() -add_library(gmock STATIC - "${googlemock_source_dir}/src/gmock-all.cc" - "${googletest_source_dir}/src/gtest-all.cc" -) -target_link_libraries(gmock ${CMAKE_THREAD_LIBS_INIT}) -add_library(gmock_main STATIC "${googlemock_source_dir}/src/gmock_main.cc") -target_link_libraries(gmock_main gmock) + set(googlemock_source_dir "${protobuf_source_dir}/third_party/googletest/googlemock") + set(googletest_source_dir "${protobuf_source_dir}/third_party/googletest/googletest") + include_directories( + ${googlemock_source_dir} + ${googletest_source_dir} + ${googletest_source_dir}/include + ${googlemock_source_dir}/include + ) + + add_library(gmock STATIC + "${googlemock_source_dir}/src/gmock-all.cc" + "${googletest_source_dir}/src/gtest-all.cc" + ) + target_link_libraries(gmock ${CMAKE_THREAD_LIBS_INIT}) + add_library(gmock_main STATIC "${googlemock_source_dir}/src/gmock_main.cc") + target_link_libraries(gmock_main gmock) + + add_library(GTest::gmock ALIAS gmock) + add_library(GTest::gmock_main ALIAS gmock_main) +endif() set(lite_test_protos google/protobuf/map_lite_unittest.proto @@ -224,7 +233,12 @@ if(MINGW) endif() add_executable(tests ${tests_files} ${common_test_files} ${tests_proto_files} ${lite_test_proto_files}) -target_link_libraries(tests libprotoc libprotobuf gmock_main) +if (MSVC) + target_compile_options(tests PRIVATE + /wd4146 # unary minus operator applied to unsigned type, result still unsigned + ) +endif() +target_link_libraries(tests libprotoc libprotobuf GTest::gmock_main) set(test_plugin_files ${protobuf_source_dir}/src/google/protobuf/compiler/mock_code_generator.cc @@ -234,21 +248,25 @@ set(test_plugin_files ) add_executable(test_plugin ${test_plugin_files}) -target_link_libraries(test_plugin libprotoc libprotobuf gmock) +target_link_libraries(test_plugin libprotoc libprotobuf GTest::gmock) set(lite_test_files ${protobuf_source_dir}/src/google/protobuf/lite_unittest.cc ) add_executable(lite-test ${lite_test_files} ${common_lite_test_files} ${lite_test_proto_files}) -target_link_libraries(lite-test libprotobuf-lite gmock_main) +target_link_libraries(lite-test libprotobuf-lite GTest::gmock_main) set(lite_arena_test_files ${protobuf_source_dir}/src/google/protobuf/lite_arena_unittest.cc ) add_executable(lite-arena-test ${lite_arena_test_files} ${common_lite_test_files} ${lite_test_proto_files}) -target_link_libraries(lite-arena-test libprotobuf-lite gmock_main) +target_link_libraries(lite-arena-test libprotobuf-lite GTest::gmock_main) add_custom_target(check COMMAND tests DEPENDS tests test_plugin WORKING_DIRECTORY ${protobuf_source_dir}) + +add_test(NAME check + COMMAND tests + WORKING_DIRECTORY "${protobuf_source_dir}") diff --git a/configure.ac b/configure.ac index b7466c28b5..4c774b0bea 100644 --- a/configure.ac +++ b/configure.ac @@ -17,7 +17,7 @@ AC_PREREQ(2.59) # In the SVN trunk, the version should always be the next anticipated release # version with the "-pre" suffix. (We used to use "-SNAPSHOT" but this pushed # the size of one file name in the dist tarfile over the 99-char limit.) -AC_INIT([Protocol Buffers],[3.18.1],[protobuf@googlegroups.com],[protobuf]) +AC_INIT([Protocol Buffers],[3.19.4],[protobuf@googlegroups.com],[protobuf]) AM_MAINTAINER_MODE([enable]) diff --git a/conformance/Makefile.am b/conformance/Makefile.am index ac6f9e1b2b..940c0a2668 100644 --- a/conformance/Makefile.am +++ b/conformance/Makefile.am @@ -316,7 +316,7 @@ conformance-java-lite: javac_middleman_lite conformance-csharp: $(other_language_protoc_outputs) @echo "Writing shortcut script conformance-csharp..." @echo '#! /bin/sh' > conformance-csharp - @echo 'dotnet ../csharp/src/Google.Protobuf.Conformance/bin/Release/netcoreapp2.1/Google.Protobuf.Conformance.dll "$$@"' >> conformance-csharp + @echo 'dotnet ../csharp/src/Google.Protobuf.Conformance/bin/Release/netcoreapp3.1/Google.Protobuf.Conformance.dll "$$@"' >> conformance-csharp @chmod +x conformance-csharp conformance-php: diff --git a/conformance/conformance_nodejs.js b/conformance/conformance_nodejs.js index 95da893f71..275fac4613 100755 --- a/conformance/conformance_nodejs.js +++ b/conformance/conformance_nodejs.js @@ -41,11 +41,17 @@ function doTest(request) { var response = new conformance.ConformanceResponse(); try { - if (request.getRequestedOutputFormat() === conformance.WireFormat.JSON) { + if (request.getRequestedOutputFormat() == conformance.WireFormat.JSON) { response.setSkipped("JSON not supported."); return response; } + if (request.getRequestedOutputFormat() == + conformance.WireFormat.TEXT_FORMAT) { + response.setSkipped('Text format is not supported as output format.'); + return response; + } + switch (request.getPayloadCase()) { case conformance.ConformanceRequest.PayloadCase.PROTOBUF_PAYLOAD: { if (request.getMessageType() == "protobuf_test_messages.proto3.TestAllTypesProto3") { @@ -67,7 +73,7 @@ function doTest(request) { } else { throw "Protobuf request doesn\'t have specific payload type"; } - } + } break; case conformance.ConformanceRequest.PayloadCase.JSON_PAYLOAD: response.setSkipped("JSON not supported."); diff --git a/conformance/conformance_python.py b/conformance/conformance_python.py index f812752440..37ee36e24c 100755 --- a/conformance/conformance_python.py +++ b/conformance/conformance_python.py @@ -39,8 +39,8 @@ import sys import os from google.protobuf import json_format from google.protobuf import message -from google3.third_party.protobuf import test_messages_proto3_pb2 -from google3.third_party.protobuf import test_messages_proto2_pb2 +from google.protobuf import test_messages_proto3_pb2 +from google.protobuf import test_messages_proto2_pb2 from google.protobuf import text_format import conformance_pb2 diff --git a/conformance/failure_list_js.txt b/conformance/failure_list_js.txt index 591cbbe9f2..b7d36b6dd0 100644 --- a/conformance/failure_list_js.txt +++ b/conformance/failure_list_js.txt @@ -1,9 +1,18 @@ +Recommended.Proto2.ProtobufInput.ValidDataRepeated.ENUM.UnpackedInput.DefaultOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.ENUM.UnpackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.ENUM.UnpackedInput.UnpackedOutput.ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataRepeated.FIXED64.PackedInput.DefaultOutput.ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataRepeated.FIXED64.PackedInput.PackedOutput.ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataRepeated.FIXED64.PackedInput.UnpackedOutput.ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataRepeated.FIXED64.UnpackedInput.DefaultOutput.ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataRepeated.FIXED64.UnpackedInput.PackedOutput.ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataRepeated.FIXED64.UnpackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.INT32.PackedInput.DefaultOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.INT32.PackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.INT32.PackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.INT32.UnpackedInput.DefaultOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.INT32.UnpackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.INT32.UnpackedInput.UnpackedOutput.ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataRepeated.INT64.PackedInput.DefaultOutput.ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataRepeated.INT64.PackedInput.PackedOutput.ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataRepeated.INT64.PackedInput.UnpackedOutput.ProtobufOutput @@ -22,23 +31,42 @@ Recommended.Proto2.ProtobufInput.ValidDataRepeated.SINT64.PackedInput.UnpackedOu Recommended.Proto2.ProtobufInput.ValidDataRepeated.SINT64.UnpackedInput.DefaultOutput.ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataRepeated.SINT64.UnpackedInput.PackedOutput.ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataRepeated.SINT64.UnpackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT32.PackedInput.DefaultOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT32.PackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT32.PackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT32.UnpackedInput.DefaultOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT32.UnpackedInput.PackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT32.UnpackedInput.UnpackedOutput.ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT64.PackedInput.DefaultOutput.ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT64.PackedInput.PackedOutput.ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT64.PackedInput.UnpackedOutput.ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT64.UnpackedInput.DefaultOutput.ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT64.UnpackedInput.PackedOutput.ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataRepeated.UINT64.UnpackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataScalarBinary.ENUM[4].ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataScalarBinary.ENUM[5].ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataScalarBinary.FIXED64[2].ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataScalarBinary.INT32[7].ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataScalarBinary.INT64[2].ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataScalarBinary.SFIXED64[2].ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataScalarBinary.SINT64[2].ProtobufOutput +Recommended.Proto2.ProtobufInput.ValidDataScalarBinary.UINT32[8].ProtobufOutput Recommended.Proto2.ProtobufInput.ValidDataScalarBinary.UINT64[2].ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.ENUM.UnpackedInput.DefaultOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.ENUM.UnpackedInput.PackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.ENUM.UnpackedInput.UnpackedOutput.ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataRepeated.FIXED64.PackedInput.DefaultOutput.ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataRepeated.FIXED64.PackedInput.PackedOutput.ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataRepeated.FIXED64.PackedInput.UnpackedOutput.ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataRepeated.FIXED64.UnpackedInput.DefaultOutput.ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataRepeated.FIXED64.UnpackedInput.PackedOutput.ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataRepeated.FIXED64.UnpackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT32.PackedInput.DefaultOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT32.PackedInput.PackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT32.PackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT32.UnpackedInput.DefaultOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT32.UnpackedInput.PackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT32.UnpackedInput.UnpackedOutput.ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT64.PackedInput.DefaultOutput.ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT64.PackedInput.PackedOutput.ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataRepeated.INT64.PackedInput.UnpackedOutput.ProtobufOutput @@ -57,48 +85,78 @@ Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT64.PackedInput.UnpackedOu Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT64.UnpackedInput.DefaultOutput.ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT64.UnpackedInput.PackedOutput.ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataRepeated.SINT64.UnpackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT32.PackedInput.DefaultOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT32.PackedInput.PackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT32.PackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT32.UnpackedInput.DefaultOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT32.UnpackedInput.PackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT32.UnpackedInput.UnpackedOutput.ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT64.PackedInput.DefaultOutput.ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT64.PackedInput.PackedOutput.ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT64.PackedInput.UnpackedOutput.ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT64.UnpackedInput.DefaultOutput.ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT64.UnpackedInput.PackedOutput.ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataRepeated.UINT64.UnpackedInput.UnpackedOutput.ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.ENUM[4].ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.ENUM[5].ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.FIXED64[2].ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.INT32[7].ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.INT64[2].ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.SFIXED64[2].ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.SINT64[2].ProtobufOutput +Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.UINT32[8].ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataScalarBinary.UINT64[2].ProtobufOutput +Required.Proto2.ProtobufInput.RepeatedScalarSelectsLast.ENUM.ProtobufOutput Required.Proto2.ProtobufInput.RepeatedScalarSelectsLast.FIXED64.ProtobufOutput Required.Proto2.ProtobufInput.RepeatedScalarSelectsLast.UINT64.ProtobufOutput +Required.Proto2.ProtobufInput.ValidDataRepeated.ENUM.UnpackedInput.ProtobufOutput Required.Proto2.ProtobufInput.ValidDataRepeated.FIXED64.PackedInput.ProtobufOutput Required.Proto2.ProtobufInput.ValidDataRepeated.FIXED64.UnpackedInput.ProtobufOutput +Required.Proto2.ProtobufInput.ValidDataRepeated.INT32.PackedInput.ProtobufOutput +Required.Proto2.ProtobufInput.ValidDataRepeated.INT32.UnpackedInput.ProtobufOutput Required.Proto2.ProtobufInput.ValidDataRepeated.INT64.PackedInput.ProtobufOutput Required.Proto2.ProtobufInput.ValidDataRepeated.INT64.UnpackedInput.ProtobufOutput Required.Proto2.ProtobufInput.ValidDataRepeated.SFIXED64.PackedInput.ProtobufOutput Required.Proto2.ProtobufInput.ValidDataRepeated.SFIXED64.UnpackedInput.ProtobufOutput Required.Proto2.ProtobufInput.ValidDataRepeated.SINT64.PackedInput.ProtobufOutput Required.Proto2.ProtobufInput.ValidDataRepeated.SINT64.UnpackedInput.ProtobufOutput +Required.Proto2.ProtobufInput.ValidDataRepeated.UINT32.PackedInput.ProtobufOutput +Required.Proto2.ProtobufInput.ValidDataRepeated.UINT32.UnpackedInput.ProtobufOutput Required.Proto2.ProtobufInput.ValidDataRepeated.UINT64.PackedInput.ProtobufOutput Required.Proto2.ProtobufInput.ValidDataRepeated.UINT64.UnpackedInput.ProtobufOutput +Required.Proto2.ProtobufInput.ValidDataScalar.ENUM[4].ProtobufOutput +Required.Proto2.ProtobufInput.ValidDataScalar.ENUM[5].ProtobufOutput Required.Proto2.ProtobufInput.ValidDataScalar.FIXED64[2].ProtobufOutput +Required.Proto2.ProtobufInput.ValidDataScalar.INT32[7].ProtobufOutput Required.Proto2.ProtobufInput.ValidDataScalar.INT64[2].ProtobufOutput Required.Proto2.ProtobufInput.ValidDataScalar.SFIXED64[2].ProtobufOutput Required.Proto2.ProtobufInput.ValidDataScalar.SINT64[2].ProtobufOutput +Required.Proto2.ProtobufInput.ValidDataScalar.UINT32[8].ProtobufOutput Required.Proto2.ProtobufInput.ValidDataScalar.UINT64[2].ProtobufOutput +Required.Proto3.ProtobufInput.RepeatedScalarSelectsLast.ENUM.ProtobufOutput Required.Proto3.ProtobufInput.RepeatedScalarSelectsLast.FIXED64.ProtobufOutput Required.Proto3.ProtobufInput.RepeatedScalarSelectsLast.UINT64.ProtobufOutput +Required.Proto3.ProtobufInput.ValidDataRepeated.ENUM.UnpackedInput.ProtobufOutput Required.Proto3.ProtobufInput.ValidDataRepeated.FIXED64.PackedInput.ProtobufOutput Required.Proto3.ProtobufInput.ValidDataRepeated.FIXED64.UnpackedInput.ProtobufOutput +Required.Proto3.ProtobufInput.ValidDataRepeated.INT32.PackedInput.ProtobufOutput +Required.Proto3.ProtobufInput.ValidDataRepeated.INT32.UnpackedInput.ProtobufOutput Required.Proto3.ProtobufInput.ValidDataRepeated.INT64.PackedInput.ProtobufOutput Required.Proto3.ProtobufInput.ValidDataRepeated.INT64.UnpackedInput.ProtobufOutput Required.Proto3.ProtobufInput.ValidDataRepeated.SFIXED64.PackedInput.ProtobufOutput Required.Proto3.ProtobufInput.ValidDataRepeated.SFIXED64.UnpackedInput.ProtobufOutput Required.Proto3.ProtobufInput.ValidDataRepeated.SINT64.PackedInput.ProtobufOutput Required.Proto3.ProtobufInput.ValidDataRepeated.SINT64.UnpackedInput.ProtobufOutput +Required.Proto3.ProtobufInput.ValidDataRepeated.UINT32.PackedInput.ProtobufOutput +Required.Proto3.ProtobufInput.ValidDataRepeated.UINT32.UnpackedInput.ProtobufOutput Required.Proto3.ProtobufInput.ValidDataRepeated.UINT64.PackedInput.ProtobufOutput Required.Proto3.ProtobufInput.ValidDataRepeated.UINT64.UnpackedInput.ProtobufOutput +Required.Proto3.ProtobufInput.ValidDataScalar.ENUM[4].ProtobufOutput +Required.Proto3.ProtobufInput.ValidDataScalar.ENUM[5].ProtobufOutput Required.Proto3.ProtobufInput.ValidDataScalar.FIXED64[2].ProtobufOutput +Required.Proto3.ProtobufInput.ValidDataScalar.INT32[7].ProtobufOutput Required.Proto3.ProtobufInput.ValidDataScalar.INT64[2].ProtobufOutput Required.Proto3.ProtobufInput.ValidDataScalar.SFIXED64[2].ProtobufOutput Required.Proto3.ProtobufInput.ValidDataScalar.SINT64[2].ProtobufOutput +Required.Proto3.ProtobufInput.ValidDataScalar.UINT32[8].ProtobufOutput Required.Proto3.ProtobufInput.ValidDataScalar.UINT64[2].ProtobufOutput diff --git a/csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs b/csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs index 1a650933ef..51fa5e01d6 100644 --- a/csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs +++ b/csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs @@ -1,698 +1,705 @@ -#region Copyright notice and license -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#endregion - -using System; -using Google.Protobuf.TestProtos; -using NUnit.Framework; -using UnitTest.Issues.TestProtos; -using Google.Protobuf.WellKnownTypes; -using Google.Protobuf.Reflection; - -using static Google.Protobuf.JsonParserTest; // For WrapInQuotes -using System.IO; -using Google.Protobuf.Collections; -using ProtobufUnittest; - -namespace Google.Protobuf -{ - /// - /// Tests for the JSON formatter. Note that in these tests, double quotes are replaced with apostrophes - /// for the sake of readability (embedding \" everywhere is painful). See the AssertJson method for details. - /// - public class JsonFormatterTest - { - [Test] - public void DefaultValues_WhenOmitted() - { - var formatter = JsonFormatter.Default; - - AssertJson("{ }", formatter.Format(new ForeignMessage())); - AssertJson("{ }", formatter.Format(new TestAllTypes())); - AssertJson("{ }", formatter.Format(new TestMap())); - } - - [Test] - public void DefaultValues_WhenIncluded() - { - var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true)); - AssertJson("{ 'c': 0 }", formatter.Format(new ForeignMessage())); - } - - [Test] - public void EnumAllowAlias() - { - var message = new TestEnumAllowAlias - { - Value = TestEnumWithDupValue.Foo2, - }; - var actualText = JsonFormatter.Default.Format(message); - var expectedText = "{ 'value': 'FOO1' }"; - AssertJson(expectedText, actualText); - } - - [Test] - public void EnumAsInt() - { - var message = new TestAllTypes - { - SingleForeignEnum = ForeignEnum.ForeignBar, - RepeatedForeignEnum = { ForeignEnum.ForeignBaz, (ForeignEnum) 100, ForeignEnum.ForeignFoo } - }; - var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatEnumsAsIntegers(true)); - var actualText = formatter.Format(message); - var expectedText = "{ " + - "'singleForeignEnum': 5, " + - "'repeatedForeignEnum': [ 6, 100, 4 ]" + - " }"; - AssertJson(expectedText, actualText); - } - - [Test] - public void AllSingleFields() - { - var message = new TestAllTypes - { - SingleBool = true, - SingleBytes = ByteString.CopyFrom(1, 2, 3, 4), - SingleDouble = 23.5, - SingleFixed32 = 23, - SingleFixed64 = 1234567890123, - SingleFloat = 12.25f, - SingleForeignEnum = ForeignEnum.ForeignBar, - SingleForeignMessage = new ForeignMessage { C = 10 }, - SingleImportEnum = ImportEnum.ImportBaz, - SingleImportMessage = new ImportMessage { D = 20 }, - SingleInt32 = 100, - SingleInt64 = 3210987654321, - SingleNestedEnum = TestAllTypes.Types.NestedEnum.Foo, - SingleNestedMessage = new TestAllTypes.Types.NestedMessage { Bb = 35 }, - SinglePublicImportMessage = new PublicImportMessage { E = 54 }, - SingleSfixed32 = -123, - SingleSfixed64 = -12345678901234, - SingleSint32 = -456, - SingleSint64 = -12345678901235, - SingleString = "test\twith\ttabs", - SingleUint32 = uint.MaxValue, - SingleUint64 = ulong.MaxValue, - }; - var actualText = JsonFormatter.Default.Format(message); - - // Fields in numeric order - var expectedText = "{ " + - "'singleInt32': 100, " + - "'singleInt64': '3210987654321', " + - "'singleUint32': 4294967295, " + - "'singleUint64': '18446744073709551615', " + - "'singleSint32': -456, " + - "'singleSint64': '-12345678901235', " + - "'singleFixed32': 23, " + - "'singleFixed64': '1234567890123', " + - "'singleSfixed32': -123, " + - "'singleSfixed64': '-12345678901234', " + - "'singleFloat': 12.25, " + - "'singleDouble': 23.5, " + - "'singleBool': true, " + - "'singleString': 'test\\twith\\ttabs', " + - "'singleBytes': 'AQIDBA==', " + - "'singleNestedMessage': { 'bb': 35 }, " + - "'singleForeignMessage': { 'c': 10 }, " + - "'singleImportMessage': { 'd': 20 }, " + - "'singleNestedEnum': 'FOO', " + - "'singleForeignEnum': 'FOREIGN_BAR', " + - "'singleImportEnum': 'IMPORT_BAZ', " + - "'singlePublicImportMessage': { 'e': 54 }" + - " }"; - AssertJson(expectedText, actualText); - } - - [Test] - public void WithFormatDefaultValues_DoesNotAffectMessageFields() - { - var message = new TestAllTypes(); - var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true)); - var json = formatter.Format(message); - Assert.IsFalse(json.Contains("\"singleNestedMessage\"")); - Assert.IsFalse(json.Contains("\"singleForeignMessage\"")); - Assert.IsFalse(json.Contains("\"singleImportMessage\"")); - } - - [Test] - public void WithFormatDefaultValues_DoesNotAffectProto3OptionalFields() - { - var message = new TestProto3Optional(); - message.OptionalInt32 = 0; - var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true)); - var json = formatter.Format(message); - // The non-optional proto3 fields are formatted, as is the optional-but-specified field. - AssertJson("{ 'optionalInt32': 0, 'singularInt32': 0, 'singularInt64': '0' }", json); - } - - [Test] - public void WithFormatDefaultValues_DoesNotAffectProto2Fields() - { - var message = new TestProtos.Proto2.ForeignMessage(); - message.C = 0; - var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true)); - var json = formatter.Format(message); - // The specified field is formatted, but the non-specified field (d) is not. - AssertJson("{ 'c': 0 }", json); - } - - [Test] - public void WithFormatDefaultValues_DoesNotAffectOneofFields() - { - var message = new TestOneof(); - var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true)); - var json = formatter.Format(message); - AssertJson("{ }", json); - } - - [Test] - public void RepeatedField() - { - AssertJson("{ 'repeatedInt32': [ 1, 2, 3, 4, 5 ] }", - JsonFormatter.Default.Format(new TestAllTypes { RepeatedInt32 = { 1, 2, 3, 4, 5 } })); - } - - [Test] - public void MapField_StringString() - { - AssertJson("{ 'mapStringString': { 'with spaces': 'bar', 'a': 'b' } }", - JsonFormatter.Default.Format(new TestMap { MapStringString = { { "with spaces", "bar" }, { "a", "b" } } })); - } - - [Test] - public void MapField_Int32Int32() - { - // The keys are quoted, but the values aren't. - AssertJson("{ 'mapInt32Int32': { '0': 1, '2': 3 } }", - JsonFormatter.Default.Format(new TestMap { MapInt32Int32 = { { 0, 1 }, { 2, 3 } } })); - } - - [Test] - public void MapField_BoolBool() - { - // The keys are quoted, but the values aren't. - AssertJson("{ 'mapBoolBool': { 'false': true, 'true': false } }", - JsonFormatter.Default.Format(new TestMap { MapBoolBool = { { false, true }, { true, false } } })); - } - - [Test] - public void NullValueOutsideStruct() - { - var message = new NullValueOutsideStruct { NullValue = NullValue.NullValue }; - AssertJson("{ 'nullValue': null }", JsonFormatter.Default.Format(message)); - } - - [Test] - public void NullValueNotInOneof() - { - var message = new NullValueNotInOneof(); - AssertJson("{ }", JsonFormatter.Default.Format(message)); - } - - [Test] - public void NullValueNotInOneof_FormatDefaults() - { - var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true)); - var message = new NullValueNotInOneof(); - AssertJson("{ 'nullValue': null }", formatter.Format(message)); - } - - [TestCase(1.0, "1")] - [TestCase(double.NaN, "'NaN'")] - [TestCase(double.PositiveInfinity, "'Infinity'")] - [TestCase(double.NegativeInfinity, "'-Infinity'")] - public void DoubleRepresentations(double value, string expectedValueText) - { - var message = new TestAllTypes { SingleDouble = value }; - string actualText = JsonFormatter.Default.Format(message); - string expectedText = "{ 'singleDouble': " + expectedValueText + " }"; - AssertJson(expectedText, actualText); - } - - [Test] - public void UnknownEnumValueNumeric_SingleField() - { - var message = new TestAllTypes { SingleForeignEnum = (ForeignEnum) 100 }; - AssertJson("{ 'singleForeignEnum': 100 }", JsonFormatter.Default.Format(message)); - } - - [Test] - public void UnknownEnumValueNumeric_RepeatedField() - { - var message = new TestAllTypes { RepeatedForeignEnum = { ForeignEnum.ForeignBaz, (ForeignEnum) 100, ForeignEnum.ForeignFoo } }; - AssertJson("{ 'repeatedForeignEnum': [ 'FOREIGN_BAZ', 100, 'FOREIGN_FOO' ] }", JsonFormatter.Default.Format(message)); - } - - [Test] - public void UnknownEnumValueNumeric_MapField() - { - var message = new TestMap { MapInt32Enum = { { 1, MapEnum.Foo }, { 2, (MapEnum) 100 }, { 3, MapEnum.Bar } } }; - AssertJson("{ 'mapInt32Enum': { '1': 'MAP_ENUM_FOO', '2': 100, '3': 'MAP_ENUM_BAR' } }", JsonFormatter.Default.Format(message)); - } - - [Test] - public void UnknownEnumValue_RepeatedField_AllEntriesUnknown() - { - var message = new TestAllTypes { RepeatedForeignEnum = { (ForeignEnum) 200, (ForeignEnum) 100 } }; - AssertJson("{ 'repeatedForeignEnum': [ 200, 100 ] }", JsonFormatter.Default.Format(message)); - } - - [Test] - [TestCase("a\u17b4b", "a\\u17b4b")] // Explicit - [TestCase("a\u0601b", "a\\u0601b")] // Ranged - [TestCase("a\u0605b", "a\u0605b")] // Passthrough (note lack of double backslash...) - public void SimpleNonAscii(string text, string encoded) - { - var message = new TestAllTypes { SingleString = text }; - AssertJson("{ 'singleString': '" + encoded + "' }", JsonFormatter.Default.Format(message)); - } - - [Test] - public void SurrogatePairEscaping() - { - var message = new TestAllTypes { SingleString = "a\uD801\uDC01b" }; - AssertJson("{ 'singleString': 'a\\ud801\\udc01b' }", JsonFormatter.Default.Format(message)); - } - - [Test] - public void InvalidSurrogatePairsFail() - { - // Note: don't use TestCase for these, as the strings can't be reliably represented - // See http://codeblog.jonskeet.uk/2014/11/07/when-is-a-string-not-a-string/ - - // Lone low surrogate - var message = new TestAllTypes { SingleString = "a\uDC01b" }; - Assert.Throws(() => JsonFormatter.Default.Format(message)); - - // Lone high surrogate - message = new TestAllTypes { SingleString = "a\uD801b" }; - Assert.Throws(() => JsonFormatter.Default.Format(message)); - } - - [Test] - [TestCase("foo_bar", "fooBar")] - [TestCase("bananaBanana", "bananaBanana")] - [TestCase("BANANABanana", "BANANABanana")] - [TestCase("simple", "simple")] - [TestCase("ACTION_AND_ADVENTURE", "ACTIONANDADVENTURE")] - [TestCase("action_and_adventure", "actionAndAdventure")] - [TestCase("kFoo", "kFoo")] - [TestCase("HTTPServer", "HTTPServer")] - [TestCase("CLIENT", "CLIENT")] - public void ToJsonName(string original, string expected) - { - Assert.AreEqual(expected, JsonFormatter.ToJsonName(original)); - } - - [Test] - [TestCase(null, "{ }")] - [TestCase("x", "{ 'fooString': 'x' }")] - [TestCase("", "{ 'fooString': '' }")] - public void Oneof(string fooStringValue, string expectedJson) - { - var message = new TestOneof(); - if (fooStringValue != null) - { - message.FooString = fooStringValue; - } - - // We should get the same result both with and without "format default values". - var formatter = JsonFormatter.Default; - AssertJson(expectedJson, formatter.Format(message)); - formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true)); - AssertJson(expectedJson, formatter.Format(message)); - } - - [Test] - public void WrapperFormatting_Single() - { - // Just a few examples, handling both classes and value types, and - // default vs non-default values - var message = new TestWellKnownTypes - { - Int64Field = 10, - Int32Field = 0, - BytesField = ByteString.FromBase64("ABCD"), - StringField = "" - }; - var expectedJson = "{ 'int64Field': '10', 'int32Field': 0, 'stringField': '', 'bytesField': 'ABCD' }"; - AssertJson(expectedJson, JsonFormatter.Default.Format(message)); - } - - [Test] - public void WrapperFormatting_Message() - { - Assert.AreEqual("\"\"", JsonFormatter.Default.Format(new StringValue())); - Assert.AreEqual("0", JsonFormatter.Default.Format(new Int32Value())); - } - - [Test] - public void WrapperFormatting_FormatDefaultValuesDoesNotFormatNull() - { - // The actual JSON here is very large because there are lots of fields. Just test a couple of them. - var message = new TestWellKnownTypes { Int32Field = 10 }; - var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true)); - var actualJson = formatter.Format(message); - // This *used* to include "int64Field": null, but that was a bug. - // WithDefaultValues should not affect message fields, including wrapper types. - Assert.IsFalse(actualJson.Contains("\"int64Field\": null")); - Assert.IsTrue(actualJson.Contains("\"int32Field\": 10")); - } - - [Test] - public void OutputIsInNumericFieldOrder_NoDefaults() - { - var formatter = JsonFormatter.Default; - var message = new TestJsonFieldOrdering { PlainString = "p1", PlainInt32 = 2 }; - AssertJson("{ 'plainString': 'p1', 'plainInt32': 2 }", formatter.Format(message)); - message = new TestJsonFieldOrdering { O1Int32 = 5, O2String = "o2", PlainInt32 = 10, PlainString = "plain" }; - AssertJson("{ 'plainString': 'plain', 'o2String': 'o2', 'plainInt32': 10, 'o1Int32': 5 }", formatter.Format(message)); - message = new TestJsonFieldOrdering { O1String = "", O2Int32 = 0, PlainInt32 = 10, PlainString = "plain" }; - AssertJson("{ 'plainString': 'plain', 'o1String': '', 'plainInt32': 10, 'o2Int32': 0 }", formatter.Format(message)); - } - - [Test] - public void OutputIsInNumericFieldOrder_WithDefaults() - { - var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true)); - var message = new TestJsonFieldOrdering(); - AssertJson("{ 'plainString': '', 'plainInt32': 0 }", formatter.Format(message)); - message = new TestJsonFieldOrdering { O1Int32 = 5, O2String = "o2", PlainInt32 = 10, PlainString = "plain" }; - AssertJson("{ 'plainString': 'plain', 'o2String': 'o2', 'plainInt32': 10, 'o1Int32': 5 }", formatter.Format(message)); - message = new TestJsonFieldOrdering { O1String = "", O2Int32 = 0, PlainInt32 = 10, PlainString = "plain" }; - AssertJson("{ 'plainString': 'plain', 'o1String': '', 'plainInt32': 10, 'o2Int32': 0 }", formatter.Format(message)); - } - - [Test] - [TestCase("1970-01-01T00:00:00Z", 0)] - [TestCase("1970-01-01T00:00:00.000000001Z", 1)] - [TestCase("1970-01-01T00:00:00.000000010Z", 10)] - [TestCase("1970-01-01T00:00:00.000000100Z", 100)] - [TestCase("1970-01-01T00:00:00.000001Z", 1000)] - [TestCase("1970-01-01T00:00:00.000010Z", 10000)] - [TestCase("1970-01-01T00:00:00.000100Z", 100000)] - [TestCase("1970-01-01T00:00:00.001Z", 1000000)] - [TestCase("1970-01-01T00:00:00.010Z", 10000000)] - [TestCase("1970-01-01T00:00:00.100Z", 100000000)] - [TestCase("1970-01-01T00:00:00.120Z", 120000000)] - [TestCase("1970-01-01T00:00:00.123Z", 123000000)] - [TestCase("1970-01-01T00:00:00.123400Z", 123400000)] - [TestCase("1970-01-01T00:00:00.123450Z", 123450000)] - [TestCase("1970-01-01T00:00:00.123456Z", 123456000)] - [TestCase("1970-01-01T00:00:00.123456700Z", 123456700)] - [TestCase("1970-01-01T00:00:00.123456780Z", 123456780)] - [TestCase("1970-01-01T00:00:00.123456789Z", 123456789)] - public void TimestampStandalone(string expected, int nanos) - { - Assert.AreEqual(WrapInQuotes(expected), new Timestamp { Nanos = nanos }.ToString()); - } - - [Test] - public void TimestampStandalone_FromDateTime() - { - // One before and one after the Unix epoch, more easily represented via DateTime. - Assert.AreEqual("\"1673-06-19T12:34:56Z\"", - new DateTime(1673, 6, 19, 12, 34, 56, DateTimeKind.Utc).ToTimestamp().ToString()); - Assert.AreEqual("\"2015-07-31T10:29:34Z\"", - new DateTime(2015, 7, 31, 10, 29, 34, DateTimeKind.Utc).ToTimestamp().ToString()); - } - - [Test] - [TestCase(-1, -1)] // Would be valid as duration - [TestCase(1, Timestamp.MaxNanos + 1)] - [TestCase(Timestamp.UnixSecondsAtBclMaxValue + 1, 0)] - [TestCase(Timestamp.UnixSecondsAtBclMinValue - 1, 0)] - public void TimestampStandalone_NonNormalized(long seconds, int nanoseconds) - { - var timestamp = new Timestamp { Seconds = seconds, Nanos = nanoseconds }; - Assert.Throws(() => JsonFormatter.Default.Format(timestamp)); - } - - [Test] - public void TimestampField() - { - var message = new TestWellKnownTypes { TimestampField = new Timestamp() }; - AssertJson("{ 'timestampField': '1970-01-01T00:00:00Z' }", JsonFormatter.Default.Format(message)); - } - - [Test] - [TestCase(0, 0, "0s")] - [TestCase(1, 0, "1s")] - [TestCase(-1, 0, "-1s")] - [TestCase(0, 1, "0.000000001s")] - [TestCase(0, 10, "0.000000010s")] - [TestCase(0, 100, "0.000000100s")] - [TestCase(0, 1000, "0.000001s")] - [TestCase(0, 10000, "0.000010s")] - [TestCase(0, 100000, "0.000100s")] - [TestCase(0, 1000000, "0.001s")] - [TestCase(0, 10000000, "0.010s")] - [TestCase(0, 100000000, "0.100s")] - [TestCase(0, 120000000, "0.120s")] - [TestCase(0, 123000000, "0.123s")] - [TestCase(0, 123400000, "0.123400s")] - [TestCase(0, 123450000, "0.123450s")] - [TestCase(0, 123456000, "0.123456s")] - [TestCase(0, 123456700, "0.123456700s")] - [TestCase(0, 123456780, "0.123456780s")] - [TestCase(0, 123456789, "0.123456789s")] - [TestCase(0, -100000000, "-0.100s")] - [TestCase(1, 100000000, "1.100s")] - [TestCase(-1, -100000000, "-1.100s")] - public void DurationStandalone(long seconds, int nanoseconds, string expected) - { - var json = JsonFormatter.Default.Format(new Duration { Seconds = seconds, Nanos = nanoseconds }); - Assert.AreEqual(WrapInQuotes(expected), json); - } - - [Test] - [TestCase(1, 2123456789)] - [TestCase(1, -100000000)] - public void DurationStandalone_NonNormalized(long seconds, int nanoseconds) - { - var duration = new Duration { Seconds = seconds, Nanos = nanoseconds }; - Assert.Throws(() => JsonFormatter.Default.Format(duration)); - } - - [Test] - public void DurationField() - { - var message = new TestWellKnownTypes { DurationField = new Duration() }; - AssertJson("{ 'durationField': '0s' }", JsonFormatter.Default.Format(message)); - } - - [Test] - public void StructSample() - { - var message = new Struct - { - Fields = - { - { "a", Value.ForNull() }, - { "b", Value.ForBool(false) }, - { "c", Value.ForNumber(10.5) }, - { "d", Value.ForString("text") }, - { "e", Value.ForList(Value.ForString("t1"), Value.ForNumber(5)) }, - { "f", Value.ForStruct(new Struct { Fields = { { "nested", Value.ForString("value") } } }) } - } - }; - AssertJson("{ 'a': null, 'b': false, 'c': 10.5, 'd': 'text', 'e': [ 't1', 5 ], 'f': { 'nested': 'value' } }", message.ToString()); - } - - [Test] - [TestCase("foo__bar")] - [TestCase("foo_3_ar")] - [TestCase("fooBar")] - public void FieldMaskInvalid(string input) - { - var mask = new FieldMask { Paths = { input } }; - Assert.Throws(() => JsonFormatter.Default.Format(mask)); - } - - [Test] - public void FieldMaskStandalone() - { - var fieldMask = new FieldMask { Paths = { "", "single", "with_underscore", "nested.field.name", "nested..double_dot" } }; - Assert.AreEqual("\",single,withUnderscore,nested.field.name,nested..doubleDot\"", fieldMask.ToString()); - - // Invalid, but we shouldn't create broken JSON... - fieldMask = new FieldMask { Paths = { "x\\y" } }; - Assert.AreEqual(@"""x\\y""", fieldMask.ToString()); - } - - [Test] - public void FieldMaskField() - { - var message = new TestWellKnownTypes { FieldMaskField = new FieldMask { Paths = { "user.display_name", "photo" } } }; - AssertJson("{ 'fieldMaskField': 'user.displayName,photo' }", JsonFormatter.Default.Format(message)); - } - - // SourceContext is an example of a well-known type with no special JSON handling - [Test] - public void SourceContextStandalone() - { - var message = new SourceContext { FileName = "foo.proto" }; - AssertJson("{ 'fileName': 'foo.proto' }", JsonFormatter.Default.Format(message)); - } - - [Test] - public void AnyWellKnownType() - { - var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithTypeRegistry(TypeRegistry.FromMessages(Timestamp.Descriptor))); - var timestamp = new DateTime(1673, 6, 19, 12, 34, 56, DateTimeKind.Utc).ToTimestamp(); - var any = Any.Pack(timestamp); - AssertJson("{ '@type': 'type.googleapis.com/google.protobuf.Timestamp', 'value': '1673-06-19T12:34:56Z' }", formatter.Format(any)); - } - - [Test] - public void AnyMessageType() - { - var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithTypeRegistry(TypeRegistry.FromMessages(TestAllTypes.Descriptor))); - var message = new TestAllTypes { SingleInt32 = 10, SingleNestedMessage = new TestAllTypes.Types.NestedMessage { Bb = 20 } }; - var any = Any.Pack(message); - AssertJson("{ '@type': 'type.googleapis.com/protobuf_unittest3.TestAllTypes', 'singleInt32': 10, 'singleNestedMessage': { 'bb': 20 } }", formatter.Format(any)); - } - - [Test] - public void AnyMessageType_CustomPrefix() - { - var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithTypeRegistry(TypeRegistry.FromMessages(TestAllTypes.Descriptor))); - var message = new TestAllTypes { SingleInt32 = 10 }; - var any = Any.Pack(message, "foo.bar/baz"); - AssertJson("{ '@type': 'foo.bar/baz/protobuf_unittest3.TestAllTypes', 'singleInt32': 10 }", formatter.Format(any)); - } - - [Test] - public void AnyNested() - { - var registry = TypeRegistry.FromMessages(TestWellKnownTypes.Descriptor, TestAllTypes.Descriptor); - var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithTypeRegistry(registry)); - - // Nest an Any as the value of an Any. - var doubleNestedMessage = new TestAllTypes { SingleInt32 = 20 }; - var nestedMessage = Any.Pack(doubleNestedMessage); - var message = new TestWellKnownTypes { AnyField = Any.Pack(nestedMessage) }; - AssertJson("{ 'anyField': { '@type': 'type.googleapis.com/google.protobuf.Any', 'value': { '@type': 'type.googleapis.com/protobuf_unittest3.TestAllTypes', 'singleInt32': 20 } } }", - formatter.Format(message)); - } - - [Test] - public void AnyUnknownType() - { - // The default type registry doesn't have any types in it. - var message = new TestAllTypes(); - var any = Any.Pack(message); - Assert.Throws(() => JsonFormatter.Default.Format(any)); - } - - [Test] - [TestCase(typeof(BoolValue), true, "true")] - [TestCase(typeof(Int32Value), 32, "32")] - [TestCase(typeof(Int64Value), 32L, "\"32\"")] - [TestCase(typeof(UInt32Value), 32U, "32")] - [TestCase(typeof(UInt64Value), 32UL, "\"32\"")] - [TestCase(typeof(StringValue), "foo", "\"foo\"")] - [TestCase(typeof(FloatValue), 1.5f, "1.5")] - [TestCase(typeof(DoubleValue), 1.5d, "1.5")] - public void Wrappers_Standalone(System.Type wrapperType, object value, string expectedJson) - { - IMessage populated = (IMessage)Activator.CreateInstance(wrapperType); - populated.Descriptor.Fields[WrappersReflection.WrapperValueFieldNumber].Accessor.SetValue(populated, value); - Assert.AreEqual(expectedJson, JsonFormatter.Default.Format(populated)); - } - - // Sanity tests for WriteValue. Not particularly comprehensive, as it's all covered above already, - // as FormatMessage uses WriteValue. - - [TestCase(null, "null")] - [TestCase(1, "1")] - [TestCase(1L, "'1'")] - [TestCase(0.5f, "0.5")] - [TestCase(0.5d, "0.5")] - [TestCase("text", "'text'")] - [TestCase("x\ny", @"'x\ny'")] - [TestCase(ForeignEnum.ForeignBar, "'FOREIGN_BAR'")] - public void WriteValue_Constant(object value, string expectedJson) - { - AssertWriteValue(value, expectedJson); - } - - [Test] - public void WriteValue_Timestamp() - { - var value = new DateTime(1673, 6, 19, 12, 34, 56, DateTimeKind.Utc).ToTimestamp(); - AssertWriteValue(value, "'1673-06-19T12:34:56Z'"); - } - - [Test] - public void WriteValue_Message() - { - var value = new TestAllTypes { SingleInt32 = 100, SingleInt64 = 3210987654321L }; - AssertWriteValue(value, "{ 'singleInt32': 100, 'singleInt64': '3210987654321' }"); - } - - [Test] - public void WriteValue_List() - { - var value = new RepeatedField { 1, 2, 3 }; - AssertWriteValue(value, "[ 1, 2, 3 ]"); - } - - [Test] - public void Proto2_DefaultValuesWritten() - { - var value = new ProtobufTestMessages.Proto2.TestAllTypesProto2() { FieldName13 = 0 }; - AssertWriteValue(value, "{ 'FieldName13': 0 }"); - } - - private static void AssertWriteValue(object value, string expectedJson) - { - var writer = new StringWriter(); - JsonFormatter.Default.WriteValue(writer, value); - string actual = writer.ToString(); - AssertJson(expectedJson, actual); - } - - /// - /// Checks that the actual JSON is the same as the expected JSON - but after replacing - /// all apostrophes in the expected JSON with double quotes. This basically makes the tests easier - /// to read. - /// - private static void AssertJson(string expectedJsonWithApostrophes, string actualJson) - { - var expectedJson = expectedJsonWithApostrophes.Replace("'", "\""); - Assert.AreEqual(expectedJson, actualJson); - } - } -} +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +using System; +using Google.Protobuf.TestProtos; +using NUnit.Framework; +using UnitTest.Issues.TestProtos; +using Google.Protobuf.WellKnownTypes; +using Google.Protobuf.Reflection; + +using static Google.Protobuf.JsonParserTest; // For WrapInQuotes +using System.IO; +using Google.Protobuf.Collections; +using ProtobufUnittest; + +namespace Google.Protobuf +{ + /// + /// Tests for the JSON formatter. Note that in these tests, double quotes are replaced with apostrophes + /// for the sake of readability (embedding \" everywhere is painful). See the AssertJson method for details. + /// + public class JsonFormatterTest + { + [Test] + public void DefaultValues_WhenOmitted() + { + var formatter = JsonFormatter.Default; + + AssertJson("{ }", formatter.Format(new ForeignMessage())); + AssertJson("{ }", formatter.Format(new TestAllTypes())); + AssertJson("{ }", formatter.Format(new TestMap())); + } + + [Test] + public void DefaultValues_WhenIncluded() + { + var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true)); + AssertJson("{ 'c': 0 }", formatter.Format(new ForeignMessage())); + } + + [Test] + public void EnumAllowAlias() + { + var message = new TestEnumAllowAlias + { + Value = TestEnumWithDupValue.Foo2, + }; + var actualText = JsonFormatter.Default.Format(message); + var expectedText = "{ 'value': 'FOO1' }"; + AssertJson(expectedText, actualText); + } + + [Test] + public void EnumAsInt() + { + var message = new TestAllTypes + { + SingleForeignEnum = ForeignEnum.ForeignBar, + RepeatedForeignEnum = { ForeignEnum.ForeignBaz, (ForeignEnum) 100, ForeignEnum.ForeignFoo } + }; + var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatEnumsAsIntegers(true)); + var actualText = formatter.Format(message); + var expectedText = "{ " + + "'singleForeignEnum': 5, " + + "'repeatedForeignEnum': [ 6, 100, 4 ]" + + " }"; + AssertJson(expectedText, actualText); + } + + [Test] + public void AllSingleFields() + { + var message = new TestAllTypes + { + SingleBool = true, + SingleBytes = ByteString.CopyFrom(1, 2, 3, 4), + SingleDouble = 23.5, + SingleFixed32 = 23, + SingleFixed64 = 1234567890123, + SingleFloat = 12.25f, + SingleForeignEnum = ForeignEnum.ForeignBar, + SingleForeignMessage = new ForeignMessage { C = 10 }, + SingleImportEnum = ImportEnum.ImportBaz, + SingleImportMessage = new ImportMessage { D = 20 }, + SingleInt32 = 100, + SingleInt64 = 3210987654321, + SingleNestedEnum = TestAllTypes.Types.NestedEnum.Foo, + SingleNestedMessage = new TestAllTypes.Types.NestedMessage { Bb = 35 }, + SinglePublicImportMessage = new PublicImportMessage { E = 54 }, + SingleSfixed32 = -123, + SingleSfixed64 = -12345678901234, + SingleSint32 = -456, + SingleSint64 = -12345678901235, + SingleString = "test\twith\ttabs", + SingleUint32 = uint.MaxValue, + SingleUint64 = ulong.MaxValue, + }; + var actualText = JsonFormatter.Default.Format(message); + + // Fields in numeric order + var expectedText = "{ " + + "'singleInt32': 100, " + + "'singleInt64': '3210987654321', " + + "'singleUint32': 4294967295, " + + "'singleUint64': '18446744073709551615', " + + "'singleSint32': -456, " + + "'singleSint64': '-12345678901235', " + + "'singleFixed32': 23, " + + "'singleFixed64': '1234567890123', " + + "'singleSfixed32': -123, " + + "'singleSfixed64': '-12345678901234', " + + "'singleFloat': 12.25, " + + "'singleDouble': 23.5, " + + "'singleBool': true, " + + "'singleString': 'test\\twith\\ttabs', " + + "'singleBytes': 'AQIDBA==', " + + "'singleNestedMessage': { 'bb': 35 }, " + + "'singleForeignMessage': { 'c': 10 }, " + + "'singleImportMessage': { 'd': 20 }, " + + "'singleNestedEnum': 'FOO', " + + "'singleForeignEnum': 'FOREIGN_BAR', " + + "'singleImportEnum': 'IMPORT_BAZ', " + + "'singlePublicImportMessage': { 'e': 54 }" + + " }"; + AssertJson(expectedText, actualText); + } + + [Test] + public void WithFormatDefaultValues_DoesNotAffectMessageFields() + { + var message = new TestAllTypes(); + var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true)); + var json = formatter.Format(message); + Assert.IsFalse(json.Contains("\"singleNestedMessage\"")); + Assert.IsFalse(json.Contains("\"singleForeignMessage\"")); + Assert.IsFalse(json.Contains("\"singleImportMessage\"")); + } + + [Test] + public void WithFormatDefaultValues_DoesNotAffectProto3OptionalFields() + { + var message = new TestProto3Optional(); + message.OptionalInt32 = 0; + var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true)); + var json = formatter.Format(message); + // The non-optional proto3 fields are formatted, as is the optional-but-specified field. + AssertJson("{ 'optionalInt32': 0, 'singularInt32': 0, 'singularInt64': '0' }", json); + } + + [Test] + public void WithFormatDefaultValues_DoesNotAffectProto2Fields() + { + var message = new TestProtos.Proto2.ForeignMessage(); + message.C = 0; + var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true)); + var json = formatter.Format(message); + // The specified field is formatted, but the non-specified field (d) is not. + AssertJson("{ 'c': 0 }", json); + } + + [Test] + public void WithFormatDefaultValues_DoesNotAffectOneofFields() + { + var message = new TestOneof(); + var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true)); + var json = formatter.Format(message); + AssertJson("{ }", json); + } + + [Test] + public void RepeatedField() + { + AssertJson("{ 'repeatedInt32': [ 1, 2, 3, 4, 5 ] }", + JsonFormatter.Default.Format(new TestAllTypes { RepeatedInt32 = { 1, 2, 3, 4, 5 } })); + } + + [Test] + public void MapField_StringString() + { + AssertJson("{ 'mapStringString': { 'with spaces': 'bar', 'a': 'b' } }", + JsonFormatter.Default.Format(new TestMap { MapStringString = { { "with spaces", "bar" }, { "a", "b" } } })); + } + + [Test] + public void MapField_Int32Int32() + { + // The keys are quoted, but the values aren't. + AssertJson("{ 'mapInt32Int32': { '0': 1, '2': 3 } }", + JsonFormatter.Default.Format(new TestMap { MapInt32Int32 = { { 0, 1 }, { 2, 3 } } })); + } + + [Test] + public void MapField_BoolBool() + { + // The keys are quoted, but the values aren't. + AssertJson("{ 'mapBoolBool': { 'false': true, 'true': false } }", + JsonFormatter.Default.Format(new TestMap { MapBoolBool = { { false, true }, { true, false } } })); + } + + [Test] + public void NullValueOutsideStruct() + { + var message = new NullValueOutsideStruct { NullValue = NullValue.NullValue }; + AssertJson("{ 'nullValue': null }", JsonFormatter.Default.Format(message)); + } + + [Test] + public void NullValueNotInOneof() + { + var message = new NullValueNotInOneof(); + AssertJson("{ }", JsonFormatter.Default.Format(message)); + } + + [Test] + public void NullValueNotInOneof_FormatDefaults() + { + var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true)); + var message = new NullValueNotInOneof(); + AssertJson("{ 'nullValue': null }", formatter.Format(message)); + } + + [TestCase(1.0, "1")] + [TestCase(double.NaN, "'NaN'")] + [TestCase(double.PositiveInfinity, "'Infinity'")] + [TestCase(double.NegativeInfinity, "'-Infinity'")] + public void DoubleRepresentations(double value, string expectedValueText) + { + var message = new TestAllTypes { SingleDouble = value }; + string actualText = JsonFormatter.Default.Format(message); + string expectedText = "{ 'singleDouble': " + expectedValueText + " }"; + AssertJson(expectedText, actualText); + } + + [Test] + public void UnknownEnumValueNumeric_SingleField() + { + var message = new TestAllTypes { SingleForeignEnum = (ForeignEnum) 100 }; + AssertJson("{ 'singleForeignEnum': 100 }", JsonFormatter.Default.Format(message)); + } + + [Test] + public void UnknownEnumValueNumeric_RepeatedField() + { + var message = new TestAllTypes { RepeatedForeignEnum = { ForeignEnum.ForeignBaz, (ForeignEnum) 100, ForeignEnum.ForeignFoo } }; + AssertJson("{ 'repeatedForeignEnum': [ 'FOREIGN_BAZ', 100, 'FOREIGN_FOO' ] }", JsonFormatter.Default.Format(message)); + } + + [Test] + public void UnknownEnumValueNumeric_MapField() + { + var message = new TestMap { MapInt32Enum = { { 1, MapEnum.Foo }, { 2, (MapEnum) 100 }, { 3, MapEnum.Bar } } }; + AssertJson("{ 'mapInt32Enum': { '1': 'MAP_ENUM_FOO', '2': 100, '3': 'MAP_ENUM_BAR' } }", JsonFormatter.Default.Format(message)); + } + + [Test] + public void UnknownEnumValue_RepeatedField_AllEntriesUnknown() + { + var message = new TestAllTypes { RepeatedForeignEnum = { (ForeignEnum) 200, (ForeignEnum) 100 } }; + AssertJson("{ 'repeatedForeignEnum': [ 200, 100 ] }", JsonFormatter.Default.Format(message)); + } + + [Test] + [TestCase("a\u17b4b", "a\\u17b4b")] // Explicit + [TestCase("a\u0601b", "a\\u0601b")] // Ranged + [TestCase("a\u0605b", "a\u0605b")] // Passthrough (note lack of double backslash...) + public void SimpleNonAscii(string text, string encoded) + { + var message = new TestAllTypes { SingleString = text }; + AssertJson("{ 'singleString': '" + encoded + "' }", JsonFormatter.Default.Format(message)); + } + + [Test] + public void SurrogatePairEscaping() + { + var message = new TestAllTypes { SingleString = "a\uD801\uDC01b" }; + AssertJson("{ 'singleString': 'a\\ud801\\udc01b' }", JsonFormatter.Default.Format(message)); + } + + [Test] + public void InvalidSurrogatePairsFail() + { + // Note: don't use TestCase for these, as the strings can't be reliably represented + // See http://codeblog.jonskeet.uk/2014/11/07/when-is-a-string-not-a-string/ + + // Lone low surrogate + var message = new TestAllTypes { SingleString = "a\uDC01b" }; + Assert.Throws(() => JsonFormatter.Default.Format(message)); + + // Lone high surrogate + message = new TestAllTypes { SingleString = "a\uD801b" }; + Assert.Throws(() => JsonFormatter.Default.Format(message)); + } + + [Test] + [TestCase("foo_bar", "fooBar")] + [TestCase("bananaBanana", "bananaBanana")] + [TestCase("BANANABanana", "BANANABanana")] + [TestCase("simple", "simple")] + [TestCase("ACTION_AND_ADVENTURE", "ACTIONANDADVENTURE")] + [TestCase("action_and_adventure", "actionAndAdventure")] + [TestCase("kFoo", "kFoo")] + [TestCase("HTTPServer", "HTTPServer")] + [TestCase("CLIENT", "CLIENT")] + public void ToJsonName(string original, string expected) + { + Assert.AreEqual(expected, JsonFormatter.ToJsonName(original)); + } + + [Test] + [TestCase(null, "{ }")] + [TestCase("x", "{ 'fooString': 'x' }")] + [TestCase("", "{ 'fooString': '' }")] + public void Oneof(string fooStringValue, string expectedJson) + { + var message = new TestOneof(); + if (fooStringValue != null) + { + message.FooString = fooStringValue; + } + + // We should get the same result both with and without "format default values". + var formatter = JsonFormatter.Default; + AssertJson(expectedJson, formatter.Format(message)); + formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true)); + AssertJson(expectedJson, formatter.Format(message)); + } + + [Test] + public void WrapperFormatting_Single() + { + // Just a few examples, handling both classes and value types, and + // default vs non-default values + var message = new TestWellKnownTypes + { + Int64Field = 10, + Int32Field = 0, + BytesField = ByteString.FromBase64("ABCD"), + StringField = "" + }; + var expectedJson = "{ 'int64Field': '10', 'int32Field': 0, 'stringField': '', 'bytesField': 'ABCD' }"; + AssertJson(expectedJson, JsonFormatter.Default.Format(message)); + } + + [Test] + public void WrapperFormatting_Message() + { + Assert.AreEqual("\"\"", JsonFormatter.Default.Format(new StringValue())); + Assert.AreEqual("0", JsonFormatter.Default.Format(new Int32Value())); + } + + [Test] + public void WrapperFormatting_FormatDefaultValuesDoesNotFormatNull() + { + // The actual JSON here is very large because there are lots of fields. Just test a couple of them. + var message = new TestWellKnownTypes { Int32Field = 10 }; + var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true)); + var actualJson = formatter.Format(message); + // This *used* to include "int64Field": null, but that was a bug. + // WithDefaultValues should not affect message fields, including wrapper types. + Assert.IsFalse(actualJson.Contains("\"int64Field\": null")); + Assert.IsTrue(actualJson.Contains("\"int32Field\": 10")); + } + + [Test] + public void OutputIsInNumericFieldOrder_NoDefaults() + { + var formatter = JsonFormatter.Default; + var message = new TestJsonFieldOrdering { PlainString = "p1", PlainInt32 = 2 }; + AssertJson("{ 'plainString': 'p1', 'plainInt32': 2 }", formatter.Format(message)); + message = new TestJsonFieldOrdering { O1Int32 = 5, O2String = "o2", PlainInt32 = 10, PlainString = "plain" }; + AssertJson("{ 'plainString': 'plain', 'o2String': 'o2', 'plainInt32': 10, 'o1Int32': 5 }", formatter.Format(message)); + message = new TestJsonFieldOrdering { O1String = "", O2Int32 = 0, PlainInt32 = 10, PlainString = "plain" }; + AssertJson("{ 'plainString': 'plain', 'o1String': '', 'plainInt32': 10, 'o2Int32': 0 }", formatter.Format(message)); + } + + [Test] + public void OutputIsInNumericFieldOrder_WithDefaults() + { + var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithFormatDefaultValues(true)); + var message = new TestJsonFieldOrdering(); + AssertJson("{ 'plainString': '', 'plainInt32': 0 }", formatter.Format(message)); + message = new TestJsonFieldOrdering { O1Int32 = 5, O2String = "o2", PlainInt32 = 10, PlainString = "plain" }; + AssertJson("{ 'plainString': 'plain', 'o2String': 'o2', 'plainInt32': 10, 'o1Int32': 5 }", formatter.Format(message)); + message = new TestJsonFieldOrdering { O1String = "", O2Int32 = 0, PlainInt32 = 10, PlainString = "plain" }; + AssertJson("{ 'plainString': 'plain', 'o1String': '', 'plainInt32': 10, 'o2Int32': 0 }", formatter.Format(message)); + } + + [Test] + [TestCase("1970-01-01T00:00:00Z", 0)] + [TestCase("1970-01-01T00:00:00.000000001Z", 1)] + [TestCase("1970-01-01T00:00:00.000000010Z", 10)] + [TestCase("1970-01-01T00:00:00.000000100Z", 100)] + [TestCase("1970-01-01T00:00:00.000001Z", 1000)] + [TestCase("1970-01-01T00:00:00.000010Z", 10000)] + [TestCase("1970-01-01T00:00:00.000100Z", 100000)] + [TestCase("1970-01-01T00:00:00.001Z", 1000000)] + [TestCase("1970-01-01T00:00:00.010Z", 10000000)] + [TestCase("1970-01-01T00:00:00.100Z", 100000000)] + [TestCase("1970-01-01T00:00:00.120Z", 120000000)] + [TestCase("1970-01-01T00:00:00.123Z", 123000000)] + [TestCase("1970-01-01T00:00:00.123400Z", 123400000)] + [TestCase("1970-01-01T00:00:00.123450Z", 123450000)] + [TestCase("1970-01-01T00:00:00.123456Z", 123456000)] + [TestCase("1970-01-01T00:00:00.123456700Z", 123456700)] + [TestCase("1970-01-01T00:00:00.123456780Z", 123456780)] + [TestCase("1970-01-01T00:00:00.123456789Z", 123456789)] + public void TimestampStandalone(string expected, int nanos) + { + Assert.AreEqual(WrapInQuotes(expected), new Timestamp { Nanos = nanos }.ToString()); + } + + [Test] + public void TimestampStandalone_FromDateTime() + { + // One before and one after the Unix epoch, more easily represented via DateTime. + Assert.AreEqual("\"1673-06-19T12:34:56Z\"", + new DateTime(1673, 6, 19, 12, 34, 56, DateTimeKind.Utc).ToTimestamp().ToString()); + Assert.AreEqual("\"2015-07-31T10:29:34Z\"", + new DateTime(2015, 7, 31, 10, 29, 34, DateTimeKind.Utc).ToTimestamp().ToString()); + } + + [Test] + [TestCase(-1, -1)] // Would be valid as duration + [TestCase(1, Timestamp.MaxNanos + 1)] + [TestCase(Timestamp.UnixSecondsAtBclMaxValue + 1, 0)] + [TestCase(Timestamp.UnixSecondsAtBclMinValue - 1, 0)] + public void TimestampStandalone_NonNormalized(long seconds, int nanoseconds) + { + var timestamp = new Timestamp { Seconds = seconds, Nanos = nanoseconds }; + Assert.Throws(() => JsonFormatter.Default.Format(timestamp)); + } + + [Test] + public void TimestampField() + { + var message = new TestWellKnownTypes { TimestampField = new Timestamp() }; + AssertJson("{ 'timestampField': '1970-01-01T00:00:00Z' }", JsonFormatter.Default.Format(message)); + } + + [Test] + [TestCase(0, 0, "0s")] + [TestCase(1, 0, "1s")] + [TestCase(-1, 0, "-1s")] + [TestCase(0, 1, "0.000000001s")] + [TestCase(0, 10, "0.000000010s")] + [TestCase(0, 100, "0.000000100s")] + [TestCase(0, 1000, "0.000001s")] + [TestCase(0, 10000, "0.000010s")] + [TestCase(0, 100000, "0.000100s")] + [TestCase(0, 1000000, "0.001s")] + [TestCase(0, 10000000, "0.010s")] + [TestCase(0, 100000000, "0.100s")] + [TestCase(0, 120000000, "0.120s")] + [TestCase(0, 123000000, "0.123s")] + [TestCase(0, 123400000, "0.123400s")] + [TestCase(0, 123450000, "0.123450s")] + [TestCase(0, 123456000, "0.123456s")] + [TestCase(0, 123456700, "0.123456700s")] + [TestCase(0, 123456780, "0.123456780s")] + [TestCase(0, 123456789, "0.123456789s")] + [TestCase(0, -100000000, "-0.100s")] + [TestCase(1, 100000000, "1.100s")] + [TestCase(-1, -100000000, "-1.100s")] + public void DurationStandalone(long seconds, int nanoseconds, string expected) + { + var json = JsonFormatter.Default.Format(new Duration { Seconds = seconds, Nanos = nanoseconds }); + Assert.AreEqual(WrapInQuotes(expected), json); + } + + [Test] + [TestCase(1, 2123456789)] + [TestCase(1, -100000000)] + public void DurationStandalone_NonNormalized(long seconds, int nanoseconds) + { + var duration = new Duration { Seconds = seconds, Nanos = nanoseconds }; + Assert.Throws(() => JsonFormatter.Default.Format(duration)); + } + + [Test] + public void DurationField() + { + var message = new TestWellKnownTypes { DurationField = new Duration() }; + AssertJson("{ 'durationField': '0s' }", JsonFormatter.Default.Format(message)); + } + + [Test] + public void StructSample() + { + var message = new Struct + { + Fields = + { + { "a", Value.ForNull() }, + { "b", Value.ForBool(false) }, + { "c", Value.ForNumber(10.5) }, + { "d", Value.ForString("text") }, + { "e", Value.ForList(Value.ForString("t1"), Value.ForNumber(5)) }, + { "f", Value.ForStruct(new Struct { Fields = { { "nested", Value.ForString("value") } } }) } + } + }; + AssertJson("{ 'a': null, 'b': false, 'c': 10.5, 'd': 'text', 'e': [ 't1', 5 ], 'f': { 'nested': 'value' } }", message.ToString()); + } + + [Test] + [TestCase("foo__bar")] + [TestCase("foo_3_ar")] + [TestCase("fooBar")] + public void FieldMaskInvalid(string input) + { + var mask = new FieldMask { Paths = { input } }; + Assert.Throws(() => JsonFormatter.Default.Format(mask)); + } + + [Test] + public void FieldMaskStandalone() + { + var fieldMask = new FieldMask { Paths = { "", "single", "with_underscore", "nested.field.name", "nested..double_dot" } }; + Assert.AreEqual("\",single,withUnderscore,nested.field.name,nested..doubleDot\"", fieldMask.ToString()); + + // Invalid, but we shouldn't create broken JSON... + fieldMask = new FieldMask { Paths = { "x\\y" } }; + Assert.AreEqual(@"""x\\y""", fieldMask.ToString()); + } + + [Test] + public void FieldMaskField() + { + var message = new TestWellKnownTypes { FieldMaskField = new FieldMask { Paths = { "user.display_name", "photo" } } }; + AssertJson("{ 'fieldMaskField': 'user.displayName,photo' }", JsonFormatter.Default.Format(message)); + } + + // SourceContext is an example of a well-known type with no special JSON handling + [Test] + public void SourceContextStandalone() + { + var message = new SourceContext { FileName = "foo.proto" }; + AssertJson("{ 'fileName': 'foo.proto' }", JsonFormatter.Default.Format(message)); + } + + [Test] + public void AnyWellKnownType() + { + var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithTypeRegistry(TypeRegistry.FromMessages(Timestamp.Descriptor))); + var timestamp = new DateTime(1673, 6, 19, 12, 34, 56, DateTimeKind.Utc).ToTimestamp(); + var any = Any.Pack(timestamp); + AssertJson("{ '@type': 'type.googleapis.com/google.protobuf.Timestamp', 'value': '1673-06-19T12:34:56Z' }", formatter.Format(any)); + } + + [Test] + public void AnyMessageType() + { + var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithTypeRegistry(TypeRegistry.FromMessages(TestAllTypes.Descriptor))); + var message = new TestAllTypes { SingleInt32 = 10, SingleNestedMessage = new TestAllTypes.Types.NestedMessage { Bb = 20 } }; + var any = Any.Pack(message); + AssertJson("{ '@type': 'type.googleapis.com/protobuf_unittest3.TestAllTypes', 'singleInt32': 10, 'singleNestedMessage': { 'bb': 20 } }", formatter.Format(any)); + } + + [Test] + public void AnyMessageType_CustomPrefix() + { + var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithTypeRegistry(TypeRegistry.FromMessages(TestAllTypes.Descriptor))); + var message = new TestAllTypes { SingleInt32 = 10 }; + var any = Any.Pack(message, "foo.bar/baz"); + AssertJson("{ '@type': 'foo.bar/baz/protobuf_unittest3.TestAllTypes', 'singleInt32': 10 }", formatter.Format(any)); + } + + [Test] + public void AnyNested() + { + var registry = TypeRegistry.FromMessages(TestWellKnownTypes.Descriptor, TestAllTypes.Descriptor); + var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithTypeRegistry(registry)); + + // Nest an Any as the value of an Any. + var doubleNestedMessage = new TestAllTypes { SingleInt32 = 20 }; + var nestedMessage = Any.Pack(doubleNestedMessage); + var message = new TestWellKnownTypes { AnyField = Any.Pack(nestedMessage) }; + AssertJson("{ 'anyField': { '@type': 'type.googleapis.com/google.protobuf.Any', 'value': { '@type': 'type.googleapis.com/protobuf_unittest3.TestAllTypes', 'singleInt32': 20 } } }", + formatter.Format(message)); + } + + [Test] + public void AnyUnknownType() + { + // The default type registry doesn't have any types in it. + var message = new TestAllTypes(); + var any = Any.Pack(message); + Assert.Throws(() => JsonFormatter.Default.Format(any)); + } + + [Test] + [TestCase(typeof(BoolValue), true, "true")] + [TestCase(typeof(Int32Value), 32, "32")] + [TestCase(typeof(Int64Value), 32L, "\"32\"")] + [TestCase(typeof(UInt32Value), 32U, "32")] + [TestCase(typeof(UInt64Value), 32UL, "\"32\"")] + [TestCase(typeof(StringValue), "foo", "\"foo\"")] + [TestCase(typeof(FloatValue), 1.5f, "1.5")] + [TestCase(typeof(DoubleValue), 1.5d, "1.5")] + public void Wrappers_Standalone(System.Type wrapperType, object value, string expectedJson) + { + IMessage populated = (IMessage)Activator.CreateInstance(wrapperType); + populated.Descriptor.Fields[WrappersReflection.WrapperValueFieldNumber].Accessor.SetValue(populated, value); + Assert.AreEqual(expectedJson, JsonFormatter.Default.Format(populated)); + } + + // Sanity tests for WriteValue. Not particularly comprehensive, as it's all covered above already, + // as FormatMessage uses WriteValue. + + [TestCase(null, "null")] + [TestCase(1, "1")] + [TestCase(1L, "'1'")] + [TestCase(0.5f, "0.5")] + [TestCase(0.5d, "0.5")] + [TestCase("text", "'text'")] + [TestCase("x\ny", @"'x\ny'")] + [TestCase(ForeignEnum.ForeignBar, "'FOREIGN_BAR'")] + public void WriteValue_Constant(object value, string expectedJson) + { + AssertWriteValue(value, expectedJson); + } + + [Test] + public void WriteValue_Timestamp() + { + var value = new DateTime(1673, 6, 19, 12, 34, 56, DateTimeKind.Utc).ToTimestamp(); + AssertWriteValue(value, "'1673-06-19T12:34:56Z'"); + } + + [Test] + public void WriteValue_Message() + { + var value = new TestAllTypes { SingleInt32 = 100, SingleInt64 = 3210987654321L }; + AssertWriteValue(value, "{ 'singleInt32': 100, 'singleInt64': '3210987654321' }"); + } + + [Test] + public void WriteValue_Message_PreserveNames() + { + var value = new TestAllTypes { SingleInt32 = 100, SingleInt64 = 3210987654321L }; + AssertWriteValue(value, "{ 'single_int32': 100, 'single_int64': '3210987654321' }", JsonFormatter.Settings.Default.WithPreserveProtoFieldNames(true)); + } + + [Test] + public void WriteValue_List() + { + var value = new RepeatedField { 1, 2, 3 }; + AssertWriteValue(value, "[ 1, 2, 3 ]"); + } + + [Test] + public void Proto2_DefaultValuesWritten() + { + var value = new ProtobufTestMessages.Proto2.TestAllTypesProto2() { FieldName13 = 0 }; + AssertWriteValue(value, "{ 'FieldName13': 0 }"); + } + + private static void AssertWriteValue(object value, string expectedJson, JsonFormatter.Settings settings = null) + { + var writer = new StringWriter(); + new JsonFormatter(settings ?? JsonFormatter.Settings.Default).WriteValue(writer, value); + string actual = writer.ToString(); + AssertJson(expectedJson, actual); + } + + /// + /// Checks that the actual JSON is the same as the expected JSON - but after replacing + /// all apostrophes in the expected JSON with double quotes. This basically makes the tests easier + /// to read. + /// + private static void AssertJson(string expectedJsonWithApostrophes, string actualJson) + { + var expectedJson = expectedJsonWithApostrophes.Replace("'", "\""); + Assert.AreEqual(expectedJson, actualJson); + } + } +} diff --git a/csharp/src/Google.Protobuf.Test/testprotos.pb b/csharp/src/Google.Protobuf.Test/testprotos.pb index 2fc4b53029..ab60703568 100644 Binary files a/csharp/src/Google.Protobuf.Test/testprotos.pb and b/csharp/src/Google.Protobuf.Test/testprotos.pb differ diff --git a/csharp/src/Google.Protobuf/JsonFormatter.cs b/csharp/src/Google.Protobuf/JsonFormatter.cs index 17fdc7f738..db7dc5c809 100644 --- a/csharp/src/Google.Protobuf/JsonFormatter.cs +++ b/csharp/src/Google.Protobuf/JsonFormatter.cs @@ -233,7 +233,14 @@ namespace Google.Protobuf writer.Write(PropertySeparator); } - WriteString(writer, accessor.Descriptor.JsonName); + if (settings.PreserveProtoFieldNames) + { + WriteString(writer, accessor.Descriptor.Name); + } + else + { + WriteString(writer, accessor.Descriptor.JsonName); + } writer.Write(NameValueSeparator); WriteValue(writer, value); @@ -816,6 +823,11 @@ namespace Google.Protobuf /// public bool FormatEnumsAsIntegers { get; } + /// + /// Whether to use the original proto field names as defined in the .proto file. Defaults to false. + /// + public bool PreserveProtoFieldNames { get; } + /// /// Creates a new object with the specified formatting of default values @@ -832,7 +844,7 @@ namespace Google.Protobuf /// /// true if default values (0, empty strings etc) should be formatted; false otherwise. /// The to use when formatting messages. - public Settings(bool formatDefaultValues, TypeRegistry typeRegistry) : this(formatDefaultValues, typeRegistry, false) + public Settings(bool formatDefaultValues, TypeRegistry typeRegistry) : this(formatDefaultValues, typeRegistry, false, false) { } @@ -842,32 +854,41 @@ namespace Google.Protobuf /// true if default values (0, empty strings etc) should be formatted; false otherwise. /// The to use when formatting messages. TypeRegistry.Empty will be used if it is null. /// true to format the enums as integers; false to format enums as enum names. + /// true to preserve proto field names; false to convert them to lowerCamelCase. private Settings(bool formatDefaultValues, TypeRegistry typeRegistry, - bool formatEnumsAsIntegers) + bool formatEnumsAsIntegers, + bool preserveProtoFieldNames) { FormatDefaultValues = formatDefaultValues; TypeRegistry = typeRegistry ?? TypeRegistry.Empty; FormatEnumsAsIntegers = formatEnumsAsIntegers; + PreserveProtoFieldNames = preserveProtoFieldNames; } /// /// Creates a new object with the specified formatting of default values and the current settings. /// /// true if default values (0, empty strings etc) should be formatted; false otherwise. - public Settings WithFormatDefaultValues(bool formatDefaultValues) => new Settings(formatDefaultValues, TypeRegistry, FormatEnumsAsIntegers); + public Settings WithFormatDefaultValues(bool formatDefaultValues) => new Settings(formatDefaultValues, TypeRegistry, FormatEnumsAsIntegers, PreserveProtoFieldNames); /// /// Creates a new object with the specified type registry and the current settings. /// /// The to use when formatting messages. - public Settings WithTypeRegistry(TypeRegistry typeRegistry) => new Settings(FormatDefaultValues, typeRegistry, FormatEnumsAsIntegers); + public Settings WithTypeRegistry(TypeRegistry typeRegistry) => new Settings(FormatDefaultValues, typeRegistry, FormatEnumsAsIntegers, PreserveProtoFieldNames); /// /// Creates a new object with the specified enums formatting option and the current settings. /// /// true to format the enums as integers; false to format enums as enum names. - public Settings WithFormatEnumsAsIntegers(bool formatEnumsAsIntegers) => new Settings(FormatDefaultValues, TypeRegistry, formatEnumsAsIntegers); + public Settings WithFormatEnumsAsIntegers(bool formatEnumsAsIntegers) => new Settings(FormatDefaultValues, TypeRegistry, formatEnumsAsIntegers, PreserveProtoFieldNames); + + /// + /// Creates a new object with the specified field name formatting option and the current settings. + /// + /// true to preserve proto field names; false to convert them to lowerCamelCase. + public Settings WithPreserveProtoFieldNames(bool preserveProtoFieldNames) => new Settings(FormatDefaultValues, TypeRegistry, FormatEnumsAsIntegers, preserveProtoFieldNames); } // Effectively a cache of mapping from enum values to the original name as specified in the proto file, diff --git a/csharp/src/Google.Protobuf/Reflection/Descriptor.cs b/csharp/src/Google.Protobuf/Reflection/Descriptor.cs index 1a9fade566..cd4d0958db 100644 --- a/csharp/src/Google.Protobuf/Reflection/Descriptor.cs +++ b/csharp/src/Google.Protobuf/Reflection/Descriptor.cs @@ -10164,8 +10164,8 @@ namespace Google.Protobuf.Reflection { /// location. /// /// Each element is a field number or an index. They form a path from - /// the root FileDescriptorProto to the place where the definition. For - /// example, this path: + /// the root FileDescriptorProto to the place where the definition occurs. + /// For example, this path: /// [ 4, 3, 2, 7, 1 ] /// refers to: /// file.message_type(3) // 4, 3 diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/Any.cs b/csharp/src/Google.Protobuf/WellKnownTypes/Any.cs index 7b5019a8ac..13066da3fe 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/Any.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/Any.cs @@ -65,7 +65,7 @@ namespace Google.Protobuf.WellKnownTypes { /// foo = any.unpack(Foo.class); /// } /// - /// Example 3: Pack and unpack a message in Python. + /// Example 3: Pack and unpack a message in Python. /// /// foo = Foo(...) /// any = Any() @@ -75,7 +75,7 @@ namespace Google.Protobuf.WellKnownTypes { /// any.Unpack(foo) /// ... /// - /// Example 4: Pack and unpack a message in Go + /// Example 4: Pack and unpack a message in Go /// /// foo := &pb.Foo{...} /// any, err := anypb.New(foo) @@ -95,7 +95,7 @@ namespace Google.Protobuf.WellKnownTypes { /// name "y.z". /// /// JSON - /// ==== + /// /// The JSON representation of an `Any` value uses the regular /// representation of the deserialized, embedded message, with an /// additional field `@type` which contains the type URL. Example: diff --git a/docs/options.md b/docs/options.md index e4ca729c0a..dbb3563af6 100644 --- a/docs/options.md +++ b/docs/options.md @@ -300,3 +300,7 @@ with info about your project (name and website) so we can add an entry for you. 1. Protoc-gen-go-svc * Website: https://github.com/dane/protoc-gen-go-svc * Extension: 1140 + +1. Embedded Proto + * Website: https://EmbeddedProto.com + * Extension: 1141 diff --git a/docs/third_party.md b/docs/third_party.md index 1bf36a8a9f..f09cff2fcb 100644 --- a/docs/third_party.md +++ b/docs/third_party.md @@ -201,3 +201,4 @@ There are miscellaneous other things you may find useful as a Protocol Buffers d * [intellij-protolint: A protobuf linter for JetBrains IDEs](https://github.com/yoheimuta/intellij-protolint) * [vim-protolint: A protobuf linter for Vim](https://github.com/yoheimuta/vim-protolint) * [super-linter: Protocol Buffer lint as GitHub Action](https://github.com/github/super-linter) +* [protoc-gen-fieldmask - A plugin to generate static type fieldmask paths](https://github.com/idodod/protoc-gen-fieldmask) diff --git a/java/core/src/main/java/com/google/protobuf/ByteString.java b/java/core/src/main/java/com/google/protobuf/ByteString.java index f9e8efd404..480f85fd6d 100644 --- a/java/core/src/main/java/com/google/protobuf/ByteString.java +++ b/java/core/src/main/java/com/google/protobuf/ByteString.java @@ -305,16 +305,14 @@ public abstract class ByteString implements Iterable, Serializable { ByteIterator latterBytes = latter.iterator(); while (formerBytes.hasNext() && latterBytes.hasNext()) { - // Note: This code was copied from com.google.common.primitives.UnsignedBytes#compare, - // as Guava libraries cannot be used in the {@code com.google.protobuf} package. int result = - Integer.compare(toInt(formerBytes.nextByte()), toInt(latterBytes.nextByte())); + Integer.valueOf(toInt(formerBytes.nextByte())) + .compareTo(toInt(latterBytes.nextByte())); if (result != 0) { return result; } } - - return Integer.compare(former.size(), latter.size()); + return Integer.valueOf(former.size()).compareTo(Integer.valueOf(latter.size())); } }; diff --git a/java/core/src/main/java/com/google/protobuf/DescriptorMessageInfoFactory.java b/java/core/src/main/java/com/google/protobuf/DescriptorMessageInfoFactory.java index 7975136596..6090985e18 100644 --- a/java/core/src/main/java/com/google/protobuf/DescriptorMessageInfoFactory.java +++ b/java/core/src/main/java/com/google/protobuf/DescriptorMessageInfoFactory.java @@ -61,8 +61,32 @@ import java.util.concurrent.ConcurrentHashMap; final class DescriptorMessageInfoFactory implements MessageInfoFactory { private static final String GET_DEFAULT_INSTANCE_METHOD_NAME = "getDefaultInstance"; private static final DescriptorMessageInfoFactory instance = new DescriptorMessageInfoFactory(); + + /** + * Names that should be avoided (in UpperCamelCase format). Using them causes the compiler to + * generate accessors whose names collide with methods defined in base classes. + * + *

Keep this list in sync with kForbiddenWordList in + * src/google/protobuf/compiler/java/java_helpers.cc + */ private static final Set specialFieldNames = - new HashSet<>(Arrays.asList("cached_size", "serialized_size", "class")); + new HashSet<>( + Arrays.asList( + // java.lang.Object: + "Class", + // com.google.protobuf.MessageLiteOrBuilder: + "DefaultInstanceForType", + // com.google.protobuf.MessageLite: + "ParserForType", + "SerializedSize", + // com.google.protobuf.MessageOrBuilder: + "AllFields", + "DescriptorForType", + "InitializationErrorString", + // TODO(b/219045204): re-enable + // "UnknownFields", + // obsolete. kept for backwards compatibility of generated code + "CachedSize")); // Disallow construction - it's a singleton. private DescriptorMessageInfoFactory() {} @@ -125,6 +149,8 @@ final class DescriptorMessageInfoFactory implements MessageInfoFactory { * *

This class is thread-safe. */ + //

The code is adapted from the C++ implementation: + // https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/compiler/java/java_helpers.h static class IsInitializedCheckAnalyzer { private final Map resultCache = @@ -593,21 +619,104 @@ final class DescriptorMessageInfoFactory implements MessageInfoFactory { String name = (fd.getType() == FieldDescriptor.Type.GROUP) ? fd.getMessageType().getName() : fd.getName(); - String suffix = specialFieldNames.contains(name) ? "__" : "_"; - return snakeCaseToCamelCase(name) + suffix; + + // convert to UpperCamelCase for comparison to the specialFieldNames + // (which are in UpperCamelCase) + String upperCamelCaseName = snakeCaseToUpperCamelCase(name); + + // Append underscores to match the behavior of the protoc java compiler + final String suffix; + if (specialFieldNames.contains(upperCamelCaseName)) { + // For proto field names that match the specialFieldNames, + // the protoc java compiler appends "__" to the java field name + // to prevent the field's accessor method names from clashing with other methods. + // For example: + // proto field name = "class" + // java field name = "class__" + // accessor method name = "getClass_()" (so that it does not clash with + // Object.getClass()) + suffix = "__"; + } else { + // For other proto field names, + // the protoc java compiler appends "_" to the java field name + // to prevent field names from clashing with java keywords. + // For example: + // proto field name = "int" + // java field name = "int_" (so that it does not clash with int keyword) + // accessor method name = "getInt()" + suffix = "_"; + } + return snakeCaseToLowerCamelCase(name) + suffix; } private static String getCachedSizeFieldName(FieldDescriptor fd) { - return snakeCaseToCamelCase(fd.getName()) + "MemoizedSerializedSize"; + return snakeCaseToLowerCamelCase(fd.getName()) + "MemoizedSerializedSize"; + } + + /** + * Converts a snake case string into lower camel case. + * + *

Some examples: + * + *

+   *     snakeCaseToLowerCamelCase("foo_bar") => "fooBar"
+   *     snakeCaseToLowerCamelCase("foo") => "foo"
+   * 
+ * + * @param snakeCase the string in snake case to convert + * @return the string converted to camel case, with a lowercase first character + */ + private static String snakeCaseToLowerCamelCase(String snakeCase) { + return snakeCaseToCamelCase(snakeCase, false); } /** - * This method must match exactly with the corresponding function in protocol compiler. See: - * https://github.com/google/protobuf/blob/v3.0.0/src/google/protobuf/compiler/java/java_helpers.cc#L153 + * Converts a snake case string into upper camel case. + * + *

Some examples: + * + *

+   *     snakeCaseToUpperCamelCase("foo_bar") => "FooBar"
+   *     snakeCaseToUpperCamelCase("foo") => "Foo"
+   * 
+ * + * @param snakeCase the string in snake case to convert + * @return the string converted to camel case, with an uppercase first character + */ + private static String snakeCaseToUpperCamelCase(String snakeCase) { + return snakeCaseToCamelCase(snakeCase, true); + } + + /** + * Converts a snake case string into camel case. + * + *

For better readability, prefer calling either {@link #snakeCaseToLowerCamelCase(String)} or + * {@link #snakeCaseToUpperCamelCase(String)}. + * + *

Some examples: + * + *

+   *     snakeCaseToCamelCase("foo_bar", false) => "fooBar"
+   *     snakeCaseToCamelCase("foo_bar", true) => "FooBar"
+   *     snakeCaseToCamelCase("foo", false) => "foo"
+   *     snakeCaseToCamelCase("foo", true) => "Foo"
+   *     snakeCaseToCamelCase("Foo", false) => "foo"
+   *     snakeCaseToCamelCase("fooBar", false) => "fooBar"
+   * 
+ * + *

This implementation of this method must exactly match the corresponding function in the + * protocol compiler. Specifically, the {@code UnderscoresToCamelCase} function in {@code + * src/google/protobuf/compiler/java/java_helpers.cc}. + * + * @param snakeCase the string in snake case to convert + * @param capFirst true if the first letter of the returned string should be uppercase. false if + * the first letter of the returned string should be lowercase. + * @return the string converted to camel case, with an uppercase or lowercase first character + * depending on if {@code capFirst} is true or false, respectively */ - private static String snakeCaseToCamelCase(String snakeCase) { + private static String snakeCaseToCamelCase(String snakeCase, boolean capFirst) { StringBuilder sb = new StringBuilder(snakeCase.length() + 1); - boolean capNext = false; + boolean capNext = capFirst; for (int ctr = 0; ctr < snakeCase.length(); ctr++) { char next = snakeCase.charAt(ctr); if (next == '_') { @@ -653,7 +762,7 @@ final class DescriptorMessageInfoFactory implements MessageInfoFactory { /** Constructs the name of the get method for the given field in the proto. */ private static String getterForField(String snakeCase) { - String camelCase = snakeCaseToCamelCase(snakeCase); + String camelCase = snakeCaseToLowerCamelCase(snakeCase); StringBuilder builder = new StringBuilder("get"); // Capitalize the first character in the field name. builder.append(Character.toUpperCase(camelCase.charAt(0))); @@ -679,7 +788,7 @@ final class DescriptorMessageInfoFactory implements MessageInfoFactory { } private static OneofInfo newInfo(Class messageType, OneofDescriptor desc) { - String camelCase = snakeCaseToCamelCase(desc.getName()); + String camelCase = snakeCaseToLowerCamelCase(desc.getName()); String valueFieldName = camelCase + "_"; String caseFieldName = camelCase + "Case_"; diff --git a/java/core/src/main/java/com/google/protobuf/Descriptors.java b/java/core/src/main/java/com/google/protobuf/Descriptors.java index 1722ba54b0..99d088ae7a 100644 --- a/java/core/src/main/java/com/google/protobuf/Descriptors.java +++ b/java/core/src/main/java/com/google/protobuf/Descriptors.java @@ -1941,7 +1941,7 @@ public final class Descriptors { new Comparator() { @Override public int compare(EnumValueDescriptor o1, EnumValueDescriptor o2) { - return Integer.compare(o1.getNumber(), o2.getNumber()); + return Integer.valueOf(o1.getNumber()).compareTo(o2.getNumber()); } }; diff --git a/java/core/src/main/java/com/google/protobuf/TextFormat.java b/java/core/src/main/java/com/google/protobuf/TextFormat.java index d9744896e2..0b15b5972e 100644 --- a/java/core/src/main/java/com/google/protobuf/TextFormat.java +++ b/java/core/src/main/java/com/google/protobuf/TextFormat.java @@ -30,8 +30,6 @@ package com.google.protobuf; -import static java.nio.charset.StandardCharsets.UTF_8; - import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.EnumDescriptor; import com.google.protobuf.Descriptors.EnumValueDescriptor; @@ -506,11 +504,11 @@ public final class TextFormat { } switch (fieldType) { case BOOLEAN: - return Boolean.compare((boolean) getKey(), (boolean) b.getKey()); + return Boolean.valueOf((boolean) getKey()).compareTo((boolean) b.getKey()); case LONG: - return Long.compare((long) getKey(), (long) b.getKey()); + return Long.valueOf((long) getKey()).compareTo((long) b.getKey()); case INT: - return Integer.compare((int) getKey(), (int) b.getKey()); + return Integer.valueOf((int) getKey()).compareTo((int) b.getKey()); case STRING: String aString = (String) getKey(); String bString = (String) b.getKey(); @@ -2408,11 +2406,12 @@ public final class TextFormat { | digitValue(input.byteAt(i + 1)) << 8 | digitValue(input.byteAt(i + 2)) << 4 | digitValue(input.byteAt(i + 3))); - if (Character.isSurrogate(ch)) { + + if (ch >= Character.MIN_SURROGATE && ch <= Character.MAX_SURROGATE) { throw new InvalidEscapeSequenceException( "Invalid escape sequence: '\\u' refers to a surrogate"); } - byte[] chUtf8 = Character.toString(ch).getBytes(UTF_8); + byte[] chUtf8 = Character.toString(ch).getBytes(Internal.UTF_8); System.arraycopy(chUtf8, 0, result, pos, chUtf8.length); pos += chUtf8.length; i += 3; @@ -2456,7 +2455,7 @@ public final class TextFormat { } int[] codepoints = new int[1]; codepoints[0] = codepoint; - byte[] chUtf8 = new String(codepoints, 0, 1).getBytes(UTF_8); + byte[] chUtf8 = new String(codepoints, 0, 1).getBytes(Internal.UTF_8); System.arraycopy(chUtf8, 0, result, pos, chUtf8.length); pos += chUtf8.length; i += 7; diff --git a/java/core/src/test/proto/com/google/protobuf/test_bad_identifiers.proto b/java/core/src/test/proto/com/google/protobuf/test_bad_identifiers.proto index e513725c04..88ea5e3cae 100644 --- a/java/core/src/test/proto/com/google/protobuf/test_bad_identifiers.proto +++ b/java/core/src/test/proto/com/google/protobuf/test_bad_identifiers.proto @@ -32,6 +32,7 @@ // This file tests that various identifiers work as field and type names even // though the same identifiers are used internally by the java code generator. +// LINT: LEGACY_NAMES syntax = "proto2"; @@ -41,10 +42,67 @@ option java_generic_services = true; // auto-added option java_package = "com.google.protobuf"; option java_outer_classname = "TestBadIdentifiersProto"; -message TestMessage { - optional string cached_size = 1; - optional string serialized_size = 2; - optional string class = 3; +// Message with field names using underscores that conflict with accessors in +// the base message class in java. See kForbiddenWordList in +// src/google/protobuf/compiler/java/java_helpers.cc +message ForbiddenWordsUnderscoreMessage { + // java.lang.Object + optional bool class = 1; + // com.google.protobuf.MessageLiteOrBuilder + optional bool default_instance_for_type = 2; + // com.google.protobuf.MessageLite + optional bool parser_for_type = 3; + optional bool serialized_size = 4; + // com.google.protobuf.MessageOrBuilder + optional bool all_fields = 5; + optional bool descriptor_for_type = 6; + optional bool initialization_error_string = 7; + // TODO(b/219045204): re-enable + // optional bool unknown_fields = 8; + // obsolete. kept for backwards compatibility of generated code + optional bool cached_size = 9; +} + +// Message with field names using leading underscores that conflict with +// accessors in the base message class in java. See kForbiddenWordList in +// src/google/protobuf/compiler/java/java_helpers.cc +message ForbiddenWordsLeadingUnderscoreMessage { + // java.lang.Object + optional bool _class = 1; + // com.google.protobuf.MessageLiteOrBuilder + optional bool _default_instance_for_type = 2; + // com.google.protobuf.MessageLite + optional bool _parser_for_type = 3; + optional bool _serialized_size = 4; + // com.google.protobuf.MessageOrBuilder + optional bool _all_fields = 5; + optional bool _descriptor_for_type = 6; + optional bool _initialization_error_string = 7; + // TODO(b/219045204): re-enable + // optional bool _unknown_fields = 8; + // obsolete. kept for backwards compatibility of generated code + optional bool _cached_size = 9; +} + +// Message with field names in camel case that conflict with accessors in the +// base message class in java. See kForbiddenWordList in +// src/google/protobuf/compiler/java/java_helpers.cc +message ForbiddenWordsCamelMessage { + // java.lang.Object + optional bool class = 1; + // com.google.protobuf.MessageLiteOrBuilder + optional bool defaultInstanceForType = 2; + // com.google.protobuf.MessageLite + optional bool serializedSize = 3; + optional bool parserForType = 4; + // com.google.protobuf.MessageOrBuilder: + optional bool initializationErrorString = 5; + optional bool descriptorForType = 6; + optional bool allFields = 7; + // TODO(b/219045204): re-enable + // optional bool unknownFields = 8; + // obsolete. kept for backwards compatibility of generated code + optional bool cachedSize = 9; } message Descriptor { @@ -82,7 +140,7 @@ message Deprecated { optional int32 field1 = 1 [deprecated = true]; optional TestEnum field2 = 2 [deprecated = true]; - optional TestMessage field3 = 3 [deprecated = true]; + optional ForbiddenWordsUnderscoreMessage field3 = 3 [deprecated = true]; } message Override { @@ -115,7 +173,8 @@ message Double { } service TestConflictingMethodNames { - rpc Override(TestMessage) returns (TestMessage); + rpc Override(ForbiddenWordsUnderscoreMessage) + returns (ForbiddenWordsUnderscoreMessage); } message TestConflictingFieldNames { @@ -123,24 +182,24 @@ message TestConflictingFieldNames { UNKNOWN = 0; FOO = 1; } - message TestMessage {} + message ForbiddenWordsUnderscoreMessage {} repeated int32 int32_field = 1; repeated TestEnum enum_field = 2; repeated string string_field = 3; repeated bytes bytes_field = 4; - repeated TestMessage message_field = 5; + repeated ForbiddenWordsUnderscoreMessage message_field = 5; optional int32 int32_field_count = 11; optional TestEnum enum_field_count = 12; optional string string_field_count = 13; optional bytes bytes_field_count = 14; - optional TestMessage message_field_count = 15; + optional ForbiddenWordsUnderscoreMessage message_field_count = 15; - repeated int32 Int32Field = 21; // NO_PROTO3 - repeated TestEnum EnumField = 22; // NO_PROTO3 - repeated string StringField = 23; // NO_PROTO3 - repeated bytes BytesField = 24; // NO_PROTO3 - repeated TestMessage MessageField = 25; // NO_PROTO3 + repeated int32 Int32Field = 21; // NO_PROTO3 + repeated TestEnum EnumField = 22; // NO_PROTO3 + repeated string StringField = 23; // NO_PROTO3 + repeated bytes BytesField = 24; // NO_PROTO3 + repeated ForbiddenWordsUnderscoreMessage MessageField = 25; // NO_PROTO3 // This field conflicts with "int32_field" as they both generate // the method getInt32FieldList(). diff --git a/java/kotlin-lite/BUILD b/java/kotlin-lite/BUILD index bfd7b8d02b..a41584a8bb 100644 --- a/java/kotlin-lite/BUILD +++ b/java/kotlin-lite/BUILD @@ -1,5 +1,6 @@ load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") load("@rules_java//java:defs.bzl", "java_lite_proto_library") +load("//:protobuf.bzl", "internal_gen_kt_protos") java_lite_proto_library( name = "example_extensible_message_java_proto_lite", @@ -47,74 +48,31 @@ java_lite_proto_library( deps = ["//java/kotlin:evil_names_proto2"], ) +internal_gen_kt_protos( + name = "gen_evil_names_proto2_lite", + deps = ["//java/kotlin:evil_names_proto2"], + lite = True, +) + java_lite_proto_library( name = "evil_names_proto3_java_proto_lite", deps = ["//java/kotlin:evil_names_proto3"], ) +internal_gen_kt_protos( + name = "gen_evil_names_proto3_lite", + deps = ["//java/kotlin:evil_names_proto3"], + lite = True, +) + java_lite_proto_library( name = "multiple_files_proto3_java_proto_lite", deps = ["//java/kotlin:multiple_files_proto3"], ) -genrule( +internal_gen_kt_protos( name = "gen_kotlin_proto3_java_multiple_files_lite", - srcs = ["src/test/proto/com/google/protobuf/multiple_files_proto3.proto"], - outs = [ - "MultipleFilesMessageALiteKt.kt", - "MultipleFilesMessageBLiteKt.kt", - "MultipleFilesProto3LiteKt.kt", - ], - cmd = "$(location //:protoc) " + - "--kotlin_out=lite:$(@D) " + - "$(location src/test/proto/com/google/protobuf/multiple_files_proto3.proto) && " + - "cp $(@D)/com/google/protobuf/kotlin/generator/MultipleFilesMessageAKt.kt " + - "$(location MultipleFilesMessageALiteKt.kt) && " + - "cp $(@D)/com/google/protobuf/kotlin/generator/MultipleFilesMessageBKt.kt " + - "$(location MultipleFilesMessageBLiteKt.kt) && " + - "cp $(@D)/com/google/protobuf/kotlin/generator/MultipleFilesProto3Kt.kt " + - "$(location MultipleFilesProto3LiteKt.kt)", - tools = ["//:protoc"], -) - -genrule( - name = "gen_evil_names_proto2_lite", - srcs = ["src/test/proto/com/google/protobuf/evil_names_proto2.proto"], - outs = [ - "EvilNamesProto2LiteKt.kt", - "HardKeywordsAllTypesProto2LiteKt.kt", - "InterfaceKt.kt", - ], - cmd = "$(location //:protoc) " + - "--kotlin_out=lite:$(@D) " + - "$(location src/test/proto/com/google/protobuf/evil_names_proto2.proto) && " + - "cp $(@D)/com/google/protobuf/kotlin/generator/EvilNamesProto2Kt.kt " + - "$(location EvilNamesProto2LiteKt.kt) && " + - "cp $(@D)/com/google/protobuf/kotlin/generator/HardKeywordsAllTypesProto2Kt.kt " + - "$(location HardKeywordsAllTypesProto2LiteKt.kt) && " + - "cp $(@D)/com/google/protobuf/kotlin/generator/InterfaceKt.kt " + - "$(location InterfaceKt.kt)", - tools = ["//:protoc"], -) - -genrule( - name = "gen_evil_names_proto3_lite", - srcs = ["src/test/proto/com/google/protobuf/evil_names_proto3.proto"], - outs = [ - "ClassKt.kt", - "EvilNamesProto3Kt.kt", - "HardKeywordsAllTypesProto3Kt.kt", - ], - cmd = "$(location //:protoc) " + - "--kotlin_out=lite:$(@D) " + - "$(location src/test/proto/com/google/protobuf/evil_names_proto3.proto) && " + - "cp $(@D)/com/google/protobuf/kotlin/generator/ClassKt.kt " + - "$(location ClassKt.kt) && " + - "cp $(@D)/com/google/protobuf/kotlin/generator/EvilNamesProto3Kt.kt " + - "$(location EvilNamesProto3Kt.kt) && " + - "cp $(@D)/com/google/protobuf/kotlin/generator/HardKeywordsAllTypesProto3Kt.kt " + - "$(location HardKeywordsAllTypesProto3Kt.kt)", - tools = ["//:protoc"], + deps = ["//java/kotlin:multiple_files_proto3"], ) kt_jvm_library( @@ -168,7 +126,7 @@ java_test( kt_jvm_library( name = "proto3_test_lite_library", - srcs = ["src/test/kotlin/com/google/protobuf/Proto3LiteTest.kt"], + srcs = ["//java/kotlin:src/test/kotlin/com/google/protobuf/Proto3Test.kt"], deps = [ ":kotlin_proto3_unittest_lite", "//java/core:test_util_lite", @@ -180,5 +138,5 @@ kt_jvm_library( java_test( name = "proto3_test_lite", runtime_deps = [":proto3_test_lite_library"], - test_class = "com.google.protobuf.kotlin.Proto3LiteTest", + test_class = "com.google.protobuf.kotlin.Proto3Test", ) diff --git a/java/kotlin-lite/pom.xml b/java/kotlin-lite/pom.xml index 1f02c4dd8c..490fbb2bfe 100644 --- a/java/kotlin-lite/pom.xml +++ b/java/kotlin-lite/pom.xml @@ -59,6 +59,7 @@ org.jetbrains.kotlin kotlin-test ${kotlin.version} + test diff --git a/java/kotlin/src/main/kotlin/com/google/protobuf/ExtendableMessageLiteExtensions.kt b/java/kotlin-lite/src/main/kotlin/com/google/protobuf/ExtendableMessageLiteExtensions.kt similarity index 100% rename from java/kotlin/src/main/kotlin/com/google/protobuf/ExtendableMessageLiteExtensions.kt rename to java/kotlin-lite/src/main/kotlin/com/google/protobuf/ExtendableMessageLiteExtensions.kt diff --git a/java/kotlin/BUILD b/java/kotlin/BUILD index 5403e6f201..b4bdf7393f 100644 --- a/java/kotlin/BUILD +++ b/java/kotlin/BUILD @@ -2,6 +2,11 @@ load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") load("@rules_java//java:defs.bzl", "java_proto_library") load("@rules_proto//proto:defs.bzl", "proto_library") load("//:protobuf_version.bzl", "PROTOBUF_VERSION") +load("//:protobuf.bzl", "internal_gen_kt_protos") + +exports_files([ + "src/test/kotlin/com/google/protobuf/Proto3Test.kt", +]) # Kotlin generated protos depend on this and only this. kt_jvm_library( @@ -35,7 +40,10 @@ kt_jvm_library( kt_jvm_library( name = "full_extensions", - srcs = ["src/main/kotlin/com/google/protobuf/ExtendableMessageExtensions.kt"], + srcs = [ + "src/main/kotlin/com/google/protobuf/Anies.kt", + "src/main/kotlin/com/google/protobuf/ExtendableMessageExtensions.kt", + ], deps = ["//java/core"], ) @@ -106,11 +114,15 @@ java_test( kt_jvm_library( name = "test_extensions_library", - srcs = ["src/test/kotlin/com/google/protobuf/ExtendableMessageExtensionsTest.kt"], + srcs = [ + "src/test/kotlin/com/google/protobuf/AniesTest.kt", + "src/test/kotlin/com/google/protobuf/ExtendableMessageExtensionsTest.kt", + ], deps = [ ":example_extensible_message_java_proto", ":full_extensions", - "//java/lite", + "//java/core:core", + ":kotlin_unittest", ":only_for_use_in_proto_generated_code_its_generator_and_tests", ":shared_runtime", "@com_github_jetbrains_kotlin//:kotlin-test", @@ -131,6 +143,16 @@ proto_library( visibility = ["//:__subpackages__"], ) +java_proto_library( + name = "evil_names_proto2_java_proto", + deps = [":evil_names_proto2"], +) + +internal_gen_kt_protos( + name = "gen_evil_names_proto2", + deps = [":evil_names_proto2"], +) + proto_library( name = "evil_names_proto3", srcs = ["src/test/proto/com/google/protobuf/evil_names_proto3.proto"], @@ -138,12 +160,12 @@ proto_library( ) java_proto_library( - name = "evil_names_proto2_java_proto", - deps = [":evil_names_proto2"], + name = "evil_names_proto3_java_proto", + deps = [":evil_names_proto3"], ) -java_proto_library( - name = "evil_names_proto3_java_proto", +internal_gen_kt_protos( + name = "gen_evil_names_proto3", deps = [":evil_names_proto3"], ) @@ -153,68 +175,14 @@ proto_library( visibility = ["//:__subpackages__"], ) -java_proto_library( name = "multiple_files_proto3_java_proto", +java_proto_library( + name = "multiple_files_proto3_java_proto", deps = [":multiple_files_proto3"], ) -genrule( +internal_gen_kt_protos( name = "gen_kotlin_proto3_java_multiple_files", - srcs = ["src/test/proto/com/google/protobuf/multiple_files_proto3.proto"], - outs = [ - "MultipleFilesMessageAKt.kt", - "MultipleFilesMessageBKt.kt", - "MultipleFilesProto3Kt.kt", - ], - cmd = "$(location //:protoc) " + - "--kotlin_out=shared,immutable:$(@D) " + - "$(location src/test/proto/com/google/protobuf/multiple_files_proto3.proto) && " + - "cp $(@D)/com/google/protobuf/kotlin/generator/MultipleFilesMessageAKt.kt " + - "$(location MultipleFilesMessageAKt.kt) && " + - "cp $(@D)/com/google/protobuf/kotlin/generator/MultipleFilesMessageBKt.kt " + - "$(location MultipleFilesMessageBKt.kt) && " + - "cp $(@D)/com/google/protobuf/kotlin/generator/MultipleFilesProto3Kt.kt " + - "$(location MultipleFilesProto3Kt.kt)", - tools = ["//:protoc"], -) - -genrule( - name = "gen_evil_names_proto2", - srcs = ["src/test/proto/com/google/protobuf/evil_names_proto2.proto"], - outs = [ - "EvilNamesProto2Kt.kt", - "HardKeywordsAllTypesProto2Kt.kt", - "InterfaceKt.kt", - ], - cmd = "$(location //:protoc) " + - "--kotlin_out=shared,immutable:$(@D) " + - "$(location src/test/proto/com/google/protobuf/evil_names_proto2.proto) && " + - "cp $(@D)/com/google/protobuf/kotlin/generator/EvilNamesProto2Kt.kt " + - "$(location EvilNamesProto2Kt.kt) && " + - "cp $(@D)/com/google/protobuf/kotlin/generator/HardKeywordsAllTypesProto2Kt.kt " + - "$(location HardKeywordsAllTypesProto2Kt.kt) && " + - "cp $(@D)/com/google/protobuf/kotlin/generator/InterfaceKt.kt " + - "$(location InterfaceKt.kt)", - tools = ["//:protoc"], -) - -genrule( - name = "gen_evil_names_proto3", - srcs = ["src/test/proto/com/google/protobuf/evil_names_proto3.proto"], - outs = [ - "ClassKt.kt", - "EvilNamesProto3Kt.kt", - "HardKeywordsAllTypesProto3Kt.kt", - ], - cmd = "$(location //:protoc) " + - "--kotlin_out=shared,immutable:$(@D) " + - "$(location src/test/proto/com/google/protobuf/evil_names_proto3.proto) && " + - "cp $(@D)/com/google/protobuf/kotlin/generator/ClassKt.kt " + - "$(location ClassKt.kt) && " + - "cp $(@D)/com/google/protobuf/kotlin/generator/EvilNamesProto3Kt.kt " + - "$(location EvilNamesProto3Kt.kt) && " + - "cp $(@D)/com/google/protobuf/kotlin/generator/HardKeywordsAllTypesProto3Kt.kt " + - "$(location HardKeywordsAllTypesProto3Kt.kt)", - tools = ["//:protoc"], + deps = [":multiple_files_proto3"], ) kt_jvm_library( @@ -228,6 +196,7 @@ kt_jvm_library( "//java/core:core", ":only_for_use_in_proto_generated_code_its_generator_and_tests", ":shared_runtime", + ":well_known_protos_kotlin", "//:java_test_protos", ], ) @@ -282,3 +251,15 @@ java_test( runtime_deps = [":proto3_test_library"], test_class = "com.google.protobuf.kotlin.Proto3Test", ) + +kt_jvm_library( + name = "well_known_protos_kotlin", + srcs = [ + "//:gen_well_known_protos_kotlin", + ], + deps = [ + "//java/core", + ":only_for_use_in_proto_generated_code_its_generator_and_tests", + ":shared_runtime", + ], +) diff --git a/java/kotlin/generate-sources-build.xml b/java/kotlin/generate-sources-build.xml index 6963f3717f..b78bc0ba7a 100644 --- a/java/kotlin/generate-sources-build.xml +++ b/java/kotlin/generate-sources-build.xml @@ -3,6 +3,7 @@ + diff --git a/java/kotlin/src/test/kotlin/com/google/protobuf/ByteStringsTest.kt b/java/kotlin/src/test/kotlin/com/google/protobuf/ByteStringsTest.kt index 99c8c9808f..e944d385aa 100644 --- a/java/kotlin/src/test/kotlin/com/google/protobuf/ByteStringsTest.kt +++ b/java/kotlin/src/test/kotlin/com/google/protobuf/ByteStringsTest.kt @@ -33,6 +33,7 @@ package com.google.protobuf.kotlin import com.google.common.truth.Truth.assertThat import com.google.protobuf.ByteString import java.lang.IndexOutOfBoundsException +import java.nio.Buffer import java.nio.ByteBuffer import kotlin.test.assertFailsWith import org.junit.Test @@ -90,8 +91,8 @@ class ByteStringsTest { @Test fun byteBufferToByteStringRespectsPositionAndLimit() { val buffer = ByteBuffer.wrap("abc".toByteArray(Charsets.UTF_8)) - buffer.position(1) - buffer.limit(2) + (buffer as java.nio.Buffer).position(1) + (buffer as java.nio.Buffer).limit(2) assertThat(buffer.toByteString()).isEqualTo(ByteString.copyFromUtf8("b")) } } diff --git a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java index 31a4a0a0ce..85b377a0eb 100644 --- a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java +++ b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java @@ -1901,7 +1901,7 @@ public class JsonFormat { return json.getAsString(); } - private ByteString parseBytes(JsonElement json) throws InvalidProtocolBufferException { + private ByteString parseBytes(JsonElement json) { try { return ByteString.copyFrom(BaseEncoding.base64().decode(json.getAsString())); } catch (IllegalArgumentException e) { diff --git a/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java b/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java index a38ebeb623..7be94dc654 100644 --- a/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java +++ b/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java @@ -1117,7 +1117,7 @@ public class JsonFormatTest { + " \"value\": \"12345\"\n" + "}"); assertRoundTripEquals(anyMessage, registry); - anyMessage = Any.pack(UInt64Value.newBuilder().setValue(12345).build()); + anyMessage = Any.pack(UInt64Value.of(12345)); assertThat(printer.print(anyMessage)) .isEqualTo( "{\n" @@ -1125,7 +1125,7 @@ public class JsonFormatTest { + " \"value\": \"12345\"\n" + "}"); assertRoundTripEquals(anyMessage, registry); - anyMessage = Any.pack(FloatValue.newBuilder().setValue(12345).build()); + anyMessage = Any.pack(FloatValue.of(12345)); assertThat(printer.print(anyMessage)) .isEqualTo( "{\n" @@ -1133,7 +1133,7 @@ public class JsonFormatTest { + " \"value\": 12345.0\n" + "}"); assertRoundTripEquals(anyMessage, registry); - anyMessage = Any.pack(DoubleValue.newBuilder().setValue(12345).build()); + anyMessage = Any.pack(DoubleValue.of(12345)); assertThat(printer.print(anyMessage)) .isEqualTo( "{\n" @@ -1340,7 +1340,7 @@ public class JsonFormatTest { } } - @Test + @Test public void testParserRejectTrailingComma() throws Exception { try { TestAllTypes.Builder builder = TestAllTypes.newBuilder(); @@ -1350,20 +1350,16 @@ public class JsonFormatTest { // Expected. } - // TODO(xiaofeng): GSON allows trailing comma in arrays even after I set - // the JsonReader to non-lenient mode. If we want to enforce strict JSON - // compliance, we might want to switch to a different JSON parser or - // implement one by ourselves. - // try { - // TestAllTypes.Builder builder = TestAllTypes.newBuilder(); - // JsonFormat.merge( - // "{\n" - // + " \"repeatedInt32\": [12345,]\n" - // + "}", builder); - // fail("Exception is expected."); - // } catch (IOException e) { - // // Expected. - // } + try { + TestAllTypes.Builder builder = TestAllTypes.newBuilder(); + mergeFromJson( + "{\n" + + " \"repeatedInt32\": [12345,]\n" + + "}", builder); + assertWithMessage("IOException expected.").fail(); + } catch (IOException e) { + // Expected. + } } @Test diff --git a/js/package.json b/js/package.json index 33f3009255..f7e8a3f1f4 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "google-protobuf", - "version": "3.18.1", + "version": "3.19.4", "description": "Protocol Buffers for JavaScript", "main": "google-protobuf.js", "files": [ diff --git a/kokoro/linux/benchmark/build.sh b/kokoro/linux/benchmark/build.sh index dee7c47ac7..f470989394 100755 --- a/kokoro/linux/benchmark/build.sh +++ b/kokoro/linux/benchmark/build.sh @@ -1,18 +1,18 @@ #!/bin/bash # +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + # Change to repo root cd $(dirname $0)/../../.. -set -ex - -# Install openJDK 11 (required by the java benchmarks) -sudo apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 78BD65473CB3BD13 -sudo add-apt-repository ppa:openjdk-r/ppa -sudo apt-get update -sudo apt-get install -y openjdk-11-jdk-headless - -# use java 11 -sudo update-java-alternatives --set /usr/lib/jvm/java-1.11.0-openjdk-amd64 -java -version - -./tests.sh benchmark +export DOCKERHUB_ORGANIZATION=protobuftesting +export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/java_stretch +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="benchmark" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/benchmark/run.sh b/kokoro/linux/benchmark/run.sh index acd8737077..963f7796ab 100755 --- a/kokoro/linux/benchmark/run.sh +++ b/kokoro/linux/benchmark/run.sh @@ -23,7 +23,7 @@ popd ./configure CXXFLAGS="-fPIC -O2" make -j8 pushd python -virtualenv -p python3 env +python3 -m venv env source env/bin/activate python3 setup.py build --cpp_implementation pip3 install --install-option="--cpp_implementation" . diff --git a/kokoro/linux/dockerfile/test/java_stretch/Dockerfile b/kokoro/linux/dockerfile/test/java_stretch/Dockerfile index 7e1feeafad..8eeb6a2d1c 100644 --- a/kokoro/linux/dockerfile/test/java_stretch/Dockerfile +++ b/kokoro/linux/dockerfile/test/java_stretch/Dockerfile @@ -11,6 +11,7 @@ RUN apt-get update && apt-get install -y \ build-essential \ bzip2 \ ccache \ + cmake \ curl \ gcc \ git \ @@ -21,6 +22,7 @@ RUN apt-get update && apt-get install -y \ libtool \ make \ parallel \ + pkg-config \ time \ wget \ # Java dependencies @@ -29,6 +31,7 @@ RUN apt-get update && apt-get install -y \ # Required for the gtest build. python2 \ # Python dependencies + python3-dev \ python3-setuptools \ python3-pip \ python3-venv \ diff --git a/kokoro/linux/dockerfile/test/python35/Dockerfile b/kokoro/linux/dockerfile/test/python35/Dockerfile deleted file mode 100644 index 50ee184536..0000000000 --- a/kokoro/linux/dockerfile/test/python35/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM python:3.5-buster - -# Install dependencies. We start with the basic ones require to build protoc -# and the C++ build -RUN apt-get update && apt-get install -y \ - autoconf \ - autotools-dev \ - build-essential \ - bzip2 \ - ccache \ - curl \ - gcc \ - git \ - libc6 \ - libc6-dbg \ - libc6-dev \ - libgtest-dev \ - libtool \ - make \ - parallel \ - time \ - wget \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Install Python libraries. -RUN python -m pip install --no-cache-dir --upgrade \ - pip \ - setuptools \ - tox \ - wheel diff --git a/kokoro/linux/dockerfile/test/python36/Dockerfile b/kokoro/linux/dockerfile/test/python36/Dockerfile deleted file mode 100644 index 742503e5a4..0000000000 --- a/kokoro/linux/dockerfile/test/python36/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM python:3.6-buster - -# Install dependencies. We start with the basic ones require to build protoc -# and the C++ build -RUN apt-get update && apt-get install -y \ - autoconf \ - autotools-dev \ - build-essential \ - bzip2 \ - ccache \ - curl \ - gcc \ - git \ - libc6 \ - libc6-dbg \ - libc6-dev \ - libgtest-dev \ - libtool \ - make \ - parallel \ - time \ - wget \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Install Python libraries. -RUN python -m pip install --no-cache-dir --upgrade \ - pip \ - setuptools \ - tox \ - wheel diff --git a/kokoro/linux/python27/build.sh b/kokoro/linux/python27/build.sh deleted file mode 100755 index 41531a15ef..0000000000 --- a/kokoro/linux/python27/build.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# -# This is the top-level script we give to Kokoro as the entry point for -# running the "pull request" project: -# -# This script selects a specific Dockerfile (for building a Docker image) and -# a script to run inside that image. Then we delegate to the general -# build_and_run_docker.sh script. - -# Change to repo root -cd $(dirname $0)/../../.. - -export DOCKERHUB_ORGANIZATION=protobuftesting -export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/python27 -export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh -export OUTPUT_DIR=testoutput -export TEST_SET="python27" -./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/python27/continuous.cfg b/kokoro/linux/python27/continuous.cfg deleted file mode 100644 index dd98469a6f..0000000000 --- a/kokoro/linux/python27/continuous.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/python27/build.sh" -timeout_mins: 120 - -action { - define_artifacts { - regex: "**/sponge_log.xml" - } -} diff --git a/kokoro/linux/python27/presubmit.cfg b/kokoro/linux/python27/presubmit.cfg deleted file mode 100644 index dd98469a6f..0000000000 --- a/kokoro/linux/python27/presubmit.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/python27/build.sh" -timeout_mins: 120 - -action { - define_artifacts { - regex: "**/sponge_log.xml" - } -} diff --git a/kokoro/linux/python27_cpp/build.sh b/kokoro/linux/python27_cpp/build.sh deleted file mode 100755 index 1a972ee9f2..0000000000 --- a/kokoro/linux/python27_cpp/build.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# -# This is the top-level script we give to Kokoro as the entry point for -# running the "pull request" project: -# -# This script selects a specific Dockerfile (for building a Docker image) and -# a script to run inside that image. Then we delegate to the general -# build_and_run_docker.sh script. - -# Change to repo root -cd $(dirname $0)/../../.. - -export DOCKERHUB_ORGANIZATION=protobuftesting -export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/python27 -export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh -export OUTPUT_DIR=testoutput -export TEST_SET="python27_cpp" -./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/python27_cpp/continuous.cfg b/kokoro/linux/python27_cpp/continuous.cfg deleted file mode 100644 index ace22d0077..0000000000 --- a/kokoro/linux/python27_cpp/continuous.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/python27_cpp/build.sh" -timeout_mins: 120 - -action { - define_artifacts { - regex: "**/sponge_log.xml" - } -} diff --git a/kokoro/linux/python27_cpp/presubmit.cfg b/kokoro/linux/python27_cpp/presubmit.cfg deleted file mode 100644 index ace22d0077..0000000000 --- a/kokoro/linux/python27_cpp/presubmit.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/python27_cpp/build.sh" -timeout_mins: 120 - -action { - define_artifacts { - regex: "**/sponge_log.xml" - } -} diff --git a/kokoro/linux/python35/build.sh b/kokoro/linux/python35/build.sh deleted file mode 100755 index 66ea03f036..0000000000 --- a/kokoro/linux/python35/build.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# -# This is the top-level script we give to Kokoro as the entry point for -# running the "pull request" project: -# -# This script selects a specific Dockerfile (for building a Docker image) and -# a script to run inside that image. Then we delegate to the general -# build_and_run_docker.sh script. - -# Change to repo root -cd $(dirname $0)/../../.. - -export DOCKERHUB_ORGANIZATION=protobuftesting -export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/python35 -export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh -export OUTPUT_DIR=testoutput -export TEST_SET="python35" -./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/python35/continuous.cfg b/kokoro/linux/python35/continuous.cfg deleted file mode 100644 index 2b3e12cbb0..0000000000 --- a/kokoro/linux/python35/continuous.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/python35/build.sh" -timeout_mins: 120 - -action { - define_artifacts { - regex: "**/sponge_log.xml" - } -} diff --git a/kokoro/linux/python35/presubmit.cfg b/kokoro/linux/python35/presubmit.cfg deleted file mode 100644 index 2b3e12cbb0..0000000000 --- a/kokoro/linux/python35/presubmit.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/python35/build.sh" -timeout_mins: 120 - -action { - define_artifacts { - regex: "**/sponge_log.xml" - } -} diff --git a/kokoro/linux/python35_cpp/build.sh b/kokoro/linux/python35_cpp/build.sh deleted file mode 100755 index 4d0dbd4986..0000000000 --- a/kokoro/linux/python35_cpp/build.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# -# This is the top-level script we give to Kokoro as the entry point for -# running the "pull request" project: -# -# This script selects a specific Dockerfile (for building a Docker image) and -# a script to run inside that image. Then we delegate to the general -# build_and_run_docker.sh script. - -# Change to repo root -cd $(dirname $0)/../../.. - -export DOCKERHUB_ORGANIZATION=protobuftesting -export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/python35 -export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh -export OUTPUT_DIR=testoutput -export TEST_SET="python35_cpp" -./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/python35_cpp/continuous.cfg b/kokoro/linux/python35_cpp/continuous.cfg deleted file mode 100644 index ad5cc8657a..0000000000 --- a/kokoro/linux/python35_cpp/continuous.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/python35_cpp/build.sh" -timeout_mins: 120 - -action { - define_artifacts { - regex: "**/sponge_log.xml" - } -} diff --git a/kokoro/linux/python35_cpp/presubmit.cfg b/kokoro/linux/python35_cpp/presubmit.cfg deleted file mode 100644 index ad5cc8657a..0000000000 --- a/kokoro/linux/python35_cpp/presubmit.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/python35_cpp/build.sh" -timeout_mins: 120 - -action { - define_artifacts { - regex: "**/sponge_log.xml" - } -} diff --git a/objectivec/GPBAny.pbobjc.h b/objectivec/GPBAny.pbobjc.h index 21b7dcf4af..5ed3f2ee68 100644 --- a/objectivec/GPBAny.pbobjc.h +++ b/objectivec/GPBAny.pbobjc.h @@ -81,7 +81,7 @@ typedef GPB_ENUM(GPBAny_FieldNumber) { * foo = any.unpack(Foo.class); * } * - * Example 3: Pack and unpack a message in Python. + * Example 3: Pack and unpack a message in Python. * * foo = Foo(...) * any = Any() @@ -91,7 +91,7 @@ typedef GPB_ENUM(GPBAny_FieldNumber) { * any.Unpack(foo) * ... * - * Example 4: Pack and unpack a message in Go + * Example 4: Pack and unpack a message in Go * * foo := &pb.Foo{...} * any, err := anypb.New(foo) @@ -112,7 +112,7 @@ typedef GPB_ENUM(GPBAny_FieldNumber) { * * * JSON - * ==== + * * The JSON representation of an `Any` value uses the regular * representation of the deserialized, embedded message, with an * additional field `\@type` which contains the type URL. Example: diff --git a/php/ext/google/protobuf/names.c b/php/ext/google/protobuf/names.c index a99188800d..9728f213c6 100644 --- a/php/ext/google/protobuf/names.c +++ b/php/ext/google/protobuf/names.c @@ -71,22 +71,23 @@ static void stringsink_uninit(stringsink *sink) { free(sink->ptr); } /* def name -> classname ******************************************************/ const char *const kReservedNames[] = { - "abstract", "and", "array", "as", "break", - "callable", "case", "catch", "class", "clone", - "const", "continue", "declare", "default", "die", - "do", "echo", "else", "elseif", "empty", - "enddeclare", "endfor", "endforeach", "endif", "endswitch", - "endwhile", "eval", "exit", "extends", "final", - "finally", "fn", "for", "foreach", "function", - "if", "implements", "include", "include_once", "instanceof", - "global", "goto", "insteadof", "interface", "isset", - "list", "match", "namespace", "new", "object", - "or", "print", "private", "protected", "public", - "require", "require_once", "return", "static", "switch", - "throw", "trait", "try", "unset", "use", - "var", "while", "xor", "yield", "int", - "float", "bool", "string", "true", "false", - "null", "void", "iterable", NULL}; + "abstract", "and", "array", "as", "break", + "callable", "case", "catch", "class", "clone", + "const", "continue", "declare", "default", "die", + "do", "echo", "else", "elseif", "empty", + "enddeclare", "endfor", "endforeach", "endif", "endswitch", + "endwhile", "eval", "exit", "extends", "final", + "finally", "fn", "for", "foreach", "function", + "if", "implements", "include", "include_once", "instanceof", + "global", "goto", "insteadof", "interface", "isset", + "list", "match", "namespace", "new", "object", + "or", "parent", "print", "private", "protected", + "public", "require", "require_once", "return", "self", + "static", "switch", "throw", "trait", "try", + "unset", "use", "var", "while", "xor", + "yield", "int", "float", "bool", "string", + "true", "false", "null", "void", "iterable", + NULL}; bool is_reserved_name(const char* name) { int i; diff --git a/php/src/Google/Protobuf/Any.php b/php/src/Google/Protobuf/Any.php index 8fdc3c483d..cc64bad3db 100644 --- a/php/src/Google/Protobuf/Any.php +++ b/php/src/Google/Protobuf/Any.php @@ -28,7 +28,7 @@ use Google\Protobuf\Internal\GPBUtil; * if (any.is(Foo.class)) { * foo = any.unpack(Foo.class); * } - * Example 3: Pack and unpack a message in Python. + * Example 3: Pack and unpack a message in Python. * foo = Foo(...) * any = Any() * any.Pack(foo) @@ -36,7 +36,7 @@ use Google\Protobuf\Internal\GPBUtil; * if any.Is(Foo.DESCRIPTOR): * any.Unpack(foo) * ... - * Example 4: Pack and unpack a message in Go + * Example 4: Pack and unpack a message in Go * foo := &pb.Foo{...} * any, err := anypb.New(foo) * if err != nil { @@ -53,7 +53,6 @@ use Google\Protobuf\Internal\GPBUtil; * in the type URL, for example "foo.bar.com/x/y.z" will yield type * name "y.z". * JSON - * ==== * The JSON representation of an `Any` value uses the regular * representation of the deserialized, embedded message, with an * additional field `@type` which contains the type URL. Example: diff --git a/php/src/Google/Protobuf/Internal/DescriptorPool.php b/php/src/Google/Protobuf/Internal/DescriptorPool.php index 2880cee641..1468a02380 100644 --- a/php/src/Google/Protobuf/Internal/DescriptorPool.php +++ b/php/src/Google/Protobuf/Internal/DescriptorPool.php @@ -159,7 +159,7 @@ class DescriptorPool if (is_null($subdesc)) { trigger_error( 'proto not added: ' . $proto - . " for " . $desc->getFullName(), E_ERROR); + . " for " . $desc->getFullName(), E_USER_ERROR); } $field->setMessageType($subdesc); break; diff --git a/php/src/Google/Protobuf/Internal/GPBUtil.php b/php/src/Google/Protobuf/Internal/GPBUtil.php index b71246e81d..4b152839ec 100644 --- a/php/src/Google/Protobuf/Internal/GPBUtil.php +++ b/php/src/Google/Protobuf/Internal/GPBUtil.php @@ -284,12 +284,12 @@ class GPBUtil "function"=>0, "global"=>0, "goto"=>0, "if"=>0, "implements"=>0, "include"=>0, "include_once"=>0, "instanceof"=>0, "insteadof"=>0, "interface"=>0, "isset"=>0, "list"=>0, "match"=>0, "namespace"=>0, - "new"=>0, "or"=>0, "print"=>0, "private"=>0, "protected"=>0, - "public"=>0, "require"=>0, "require_once"=>0, "return"=>0, - "static"=>0, "switch"=>0, "throw"=>0, "trait"=>0, "try"=>0, - "unset"=>0, "use"=>0, "var"=>0, "while"=>0, "xor"=>0, "yield"=>0, - "int"=>0, "float"=>0, "bool"=>0, "string"=>0, "true"=>0, "false"=>0, - "null"=>0, "void"=>0, "iterable"=>0 + "new"=>0, "or"=>0, "parent"=>0, "print"=>0, "private"=>0, + "protected"=>0,"public"=>0, "require"=>0, "require_once"=>0, + "return"=>0, "self"=>0, "static"=>0, "switch"=>0, "throw"=>0, + "trait"=>0, "try"=>0,"unset"=>0, "use"=>0, "var"=>0, "while"=>0, + "xor"=>0, "yield"=>0, "int"=>0, "float"=>0, "bool"=>0, "string"=>0, + "true"=>0, "false"=>0, "null"=>0, "void"=>0, "iterable"=>0 ); if (array_key_exists(strtolower($classname), $reserved_words)) { diff --git a/php/src/Google/Protobuf/Internal/MapField.php b/php/src/Google/Protobuf/Internal/MapField.php index 86463a9a47..d44377a179 100644 --- a/php/src/Google/Protobuf/Internal/MapField.php +++ b/php/src/Google/Protobuf/Internal/MapField.php @@ -135,6 +135,7 @@ class MapField implements \ArrayAccess, \IteratorAggregate, \Countable * @return object The stored element at given key. * @throws \ErrorException Invalid type for index. * @throws \ErrorException Non-existing index. + * @todo need to add return type mixed (require update php version to 8.0) */ #[\ReturnTypeWillChange] public function offsetGet($key) @@ -153,6 +154,7 @@ class MapField implements \ArrayAccess, \IteratorAggregate, \Countable * @throws \ErrorException Invalid type for key. * @throws \ErrorException Invalid type for value. * @throws \ErrorException Non-existing key. + * @todo need to add return type void (require update php version to 7.1) */ #[\ReturnTypeWillChange] public function offsetSet($key, $value) @@ -212,6 +214,7 @@ class MapField implements \ArrayAccess, \IteratorAggregate, \Countable * @param object $key The key of the element to be removed. * @return void * @throws \ErrorException Invalid type for key. + * @todo need to add return type void (require update php version to 7.1) */ #[\ReturnTypeWillChange] public function offsetUnset($key) diff --git a/php/src/Google/Protobuf/Internal/MapFieldIter.php b/php/src/Google/Protobuf/Internal/MapFieldIter.php index a3c834bd2c..2ff6b44ae3 100644 --- a/php/src/Google/Protobuf/Internal/MapFieldIter.php +++ b/php/src/Google/Protobuf/Internal/MapFieldIter.php @@ -67,6 +67,7 @@ class MapFieldIter implements \Iterator * Reset the status of the iterator * * @return void + * @todo need to add return type void (require update php version to 7.1) */ #[\ReturnTypeWillChange] public function rewind() @@ -78,6 +79,7 @@ class MapFieldIter implements \Iterator * Return the element at the current position. * * @return object The element at the current position. + * @todo need to add return type mixed (require update php version to 8.0) */ #[\ReturnTypeWillChange] public function current() @@ -89,6 +91,7 @@ class MapFieldIter implements \Iterator * Return the current key. * * @return object The current key. + * @todo need to add return type mixed (require update php version to 8.0) */ #[\ReturnTypeWillChange] public function key() @@ -119,6 +122,7 @@ class MapFieldIter implements \Iterator * Move to the next position. * * @return void + * @todo need to add return type void (require update php version to 7.1) */ #[\ReturnTypeWillChange] public function next() diff --git a/php/src/Google/Protobuf/Internal/Message.php b/php/src/Google/Protobuf/Internal/Message.php index 19b48f0b50..1d1fbf27c5 100644 --- a/php/src/Google/Protobuf/Internal/Message.php +++ b/php/src/Google/Protobuf/Internal/Message.php @@ -423,7 +423,7 @@ class Message } break; case GPBType::GROUP: - trigger_error("Not implemented.", E_ERROR); + trigger_error("Not implemented.", E_USER_ERROR); break; case GPBType::MESSAGE: if ($field->isMap()) { diff --git a/php/src/Google/Protobuf/Internal/RepeatedField.php b/php/src/Google/Protobuf/Internal/RepeatedField.php index 704123ad15..cfe5140d77 100644 --- a/php/src/Google/Protobuf/Internal/RepeatedField.php +++ b/php/src/Google/Protobuf/Internal/RepeatedField.php @@ -121,6 +121,7 @@ class RepeatedField implements \ArrayAccess, \IteratorAggregate, \Countable * @return object The stored element at given index. * @throws \ErrorException Invalid type for index. * @throws \ErrorException Non-existing index. + * @todo need to add return type mixed (require update php version to 8.0) */ #[\ReturnTypeWillChange] public function offsetGet($offset) @@ -139,6 +140,7 @@ class RepeatedField implements \ArrayAccess, \IteratorAggregate, \Countable * @throws \ErrorException Invalid type for index. * @throws \ErrorException Non-existing index. * @throws \ErrorException Incorrect type of the element. + * @todo need to add return type void (require update php version to 7.1) */ #[\ReturnTypeWillChange] public function offsetSet($offset, $value) @@ -211,6 +213,7 @@ class RepeatedField implements \ArrayAccess, \IteratorAggregate, \Countable * @throws \ErrorException Invalid type for index. * @throws \ErrorException The element to be removed is not at the end of the * RepeatedField. + * @todo need to add return type void (require update php version to 7.1) */ #[\ReturnTypeWillChange] public function offsetUnset($offset) diff --git a/php/src/Google/Protobuf/Internal/RepeatedFieldIter.php b/php/src/Google/Protobuf/Internal/RepeatedFieldIter.php index 3c85869989..ec99b64ae7 100644 --- a/php/src/Google/Protobuf/Internal/RepeatedFieldIter.php +++ b/php/src/Google/Protobuf/Internal/RepeatedFieldIter.php @@ -70,6 +70,7 @@ class RepeatedFieldIter implements \Iterator * Reset the status of the iterator * * @return void + * @todo need to add return type void (require update php version to 7.1) */ #[\ReturnTypeWillChange] public function rewind() @@ -81,6 +82,7 @@ class RepeatedFieldIter implements \Iterator * Return the element at the current position. * * @return object The element at the current position. + * @todo need to add return type mixed (require update php version to 8.0) */ #[\ReturnTypeWillChange] public function current() @@ -92,6 +94,7 @@ class RepeatedFieldIter implements \Iterator * Return the current position. * * @return integer The current position. + * @todo need to add return type mixed (require update php version to 8.0) */ #[\ReturnTypeWillChange] public function key() @@ -103,6 +106,7 @@ class RepeatedFieldIter implements \Iterator * Move to the next position. * * @return void + * @todo need to add return type void (require update php version to 7.1) */ #[\ReturnTypeWillChange] public function next() diff --git a/php/src/Google/Protobuf/Internal/SourceCodeInfo/Location.php b/php/src/Google/Protobuf/Internal/SourceCodeInfo/Location.php index c4cc667e3a..a3e8e721a2 100644 --- a/php/src/Google/Protobuf/Internal/SourceCodeInfo/Location.php +++ b/php/src/Google/Protobuf/Internal/SourceCodeInfo/Location.php @@ -19,8 +19,8 @@ class Location extends \Google\Protobuf\Internal\Message * Identifies which part of the FileDescriptorProto was defined at this * location. * Each element is a field number or an index. They form a path from - * the root FileDescriptorProto to the place where the definition. For - * example, this path: + * the root FileDescriptorProto to the place where the definition occurs. + * For example, this path: * [ 4, 3, 2, 7, 1 ] * refers to: * file.message_type(3) // 4, 3 @@ -111,8 +111,8 @@ class Location extends \Google\Protobuf\Internal\Message * Identifies which part of the FileDescriptorProto was defined at this * location. * Each element is a field number or an index. They form a path from - * the root FileDescriptorProto to the place where the definition. For - * example, this path: + * the root FileDescriptorProto to the place where the definition occurs. + * For example, this path: * [ 4, 3, 2, 7, 1 ] * refers to: * file.message_type(3) // 4, 3 @@ -185,8 +185,8 @@ class Location extends \Google\Protobuf\Internal\Message * Identifies which part of the FileDescriptorProto was defined at this * location. * Each element is a field number or an index. They form a path from - * the root FileDescriptorProto to the place where the definition. For - * example, this path: + * the root FileDescriptorProto to the place where the definition occurs. + * For example, this path: * [ 4, 3, 2, 7, 1 ] * refers to: * file.message_type(3) // 4, 3 @@ -216,8 +216,8 @@ class Location extends \Google\Protobuf\Internal\Message * Identifies which part of the FileDescriptorProto was defined at this * location. * Each element is a field number or an index. They form a path from - * the root FileDescriptorProto to the place where the definition. For - * example, this path: + * the root FileDescriptorProto to the place where the definition occurs. + * For example, this path: * [ 4, 3, 2, 7, 1 ] * refers to: * file.message_type(3) // 4, 3 diff --git a/php/tests/GeneratedClassTest.php b/php/tests/GeneratedClassTest.php index a6101dcbfb..42b1f7458b 100644 --- a/php/tests/GeneratedClassTest.php +++ b/php/tests/GeneratedClassTest.php @@ -934,6 +934,7 @@ class GeneratedClassTest extends TestBase $m = new \Lower\PBnamespace(); $m = new \Lower\PBnew(); $m = new \Lower\PBor(); + $m = new \Lower\PBparent(); $m = new \Lower\PBprint(); $m = new \Lower\PBprivate(); $m = new \Lower\PBprotected(); @@ -941,6 +942,7 @@ class GeneratedClassTest extends TestBase $m = new \Lower\PBrequire(); $m = new \Lower\PBrequire_once(); $m = new \Lower\PBreturn(); + $m = new \Lower\PBself(); $m = new \Lower\PBstatic(); $m = new \Lower\PBswitch(); $m = new \Lower\PBthrow(); @@ -1012,6 +1014,7 @@ class GeneratedClassTest extends TestBase $m = new \Upper\PBNAMESPACE(); $m = new \Upper\PBNEW(); $m = new \Upper\PBOR(); + $m = new \Upper\PBPARENT(); $m = new \Upper\PBPRINT(); $m = new \Upper\PBPRIVATE(); $m = new \Upper\PBPROTECTED(); @@ -1019,6 +1022,7 @@ class GeneratedClassTest extends TestBase $m = new \Upper\PBREQUIRE(); $m = new \Upper\PBREQUIRE_ONCE(); $m = new \Upper\PBRETURN(); + $m = new \Upper\PBSELF(); $m = new \Upper\PBSTATIC(); $m = new \Upper\PBSWITCH(); $m = new \Upper\PBTHROW(); @@ -1090,6 +1094,7 @@ class GeneratedClassTest extends TestBase $m = new \Lower_enum\PBnamespace(); $m = new \Lower_enum\PBnew(); $m = new \Lower_enum\PBor(); + $m = new \Lower_enum\PBparent(); $m = new \Lower_enum\PBprint(); $m = new \Lower_enum\PBprivate(); $m = new \Lower_enum\PBprotected(); @@ -1097,6 +1102,7 @@ class GeneratedClassTest extends TestBase $m = new \Lower_enum\PBrequire(); $m = new \Lower_enum\PBrequire_once(); $m = new \Lower_enum\PBreturn(); + $m = new \Lower_enum\PBself(); $m = new \Lower_enum\PBstatic(); $m = new \Lower_enum\PBswitch(); $m = new \Lower_enum\PBthrow(); @@ -1168,6 +1174,7 @@ class GeneratedClassTest extends TestBase $m = new \Upper_enum\PBNAMESPACE(); $m = new \Upper_enum\PBNEW(); $m = new \Upper_enum\PBOR(); + $m = new \Upper_enum\PBPARENT(); $m = new \Upper_enum\PBPRINT(); $m = new \Upper_enum\PBPRIVATE(); $m = new \Upper_enum\PBPROTECTED(); @@ -1175,6 +1182,7 @@ class GeneratedClassTest extends TestBase $m = new \Upper_enum\PBREQUIRE(); $m = new \Upper_enum\PBREQUIRE_ONCE(); $m = new \Upper_enum\PBRETURN(); + $m = new \Upper_enum\PBSELF(); $m = new \Upper_enum\PBSTATIC(); $m = new \Upper_enum\PBSWITCH(); $m = new \Upper_enum\PBTHROW(); @@ -1273,6 +1281,8 @@ class GeneratedClassTest extends TestBase $m = \Lower_enum_value\NotAllowed::null; $m = \Lower_enum_value\NotAllowed::void; $m = \Lower_enum_value\NotAllowed::iterable; + $m = \Lower_enum_value\NotAllowed::parent; + $m = \Lower_enum_value\NotAllowed::self; $m = \Upper_enum_value\NotAllowed::PBABSTRACT; $m = \Upper_enum_value\NotAllowed::PBAND; @@ -1351,6 +1361,8 @@ class GeneratedClassTest extends TestBase $m = \Upper_enum_value\NotAllowed::NULL; $m = \Upper_enum_value\NotAllowed::VOID; $m = \Upper_enum_value\NotAllowed::ITERABLE; + $m = \Upper_enum_value\NotAllowed::PARENT; + $m = \Upper_enum_value\NotAllowed::SELF; $this->assertTrue(true); } diff --git a/php/tests/proto/test_reserved_enum_lower.proto b/php/tests/proto/test_reserved_enum_lower.proto index 60807d97cb..f8557d250f 100644 --- a/php/tests/proto/test_reserved_enum_lower.proto +++ b/php/tests/proto/test_reserved_enum_lower.proto @@ -52,6 +52,7 @@ enum match { ZERO47 = 0; } enum namespace { ZERO48 = 0; } enum new { ZERO49 = 0; } enum or { ZERO50 = 0; } +enum parent { ZERO78 = 0; } enum print { ZERO51 = 0; } enum private { ZERO52 = 0; } enum protected { ZERO53 = 0; } @@ -59,6 +60,7 @@ enum public { ZERO54 = 0; } enum require { ZERO55 = 0; } enum require_once { ZERO56 = 0; } enum return { ZERO57 = 0; } +enum self { ZERO79 = 0; } enum static { ZERO58 = 0; } enum switch { ZERO59 = 0; } enum throw { ZERO60 = 0; } diff --git a/php/tests/proto/test_reserved_enum_upper.proto b/php/tests/proto/test_reserved_enum_upper.proto index a107332918..8d382ab31e 100644 --- a/php/tests/proto/test_reserved_enum_upper.proto +++ b/php/tests/proto/test_reserved_enum_upper.proto @@ -52,6 +52,7 @@ enum MATCH { ZERO47 = 0; } enum NAMESPACE { ZERO48 = 0; } enum NEW { ZERO49 = 0; } enum OR { ZERO50 = 0; } +enum PARENT { ZERO78 = 0; } enum PRINT { ZERO51 = 0; } enum PRIVATE { ZERO52 = 0; } enum PROTECTED { ZERO53 = 0; } @@ -59,6 +60,7 @@ enum PUBLIC { ZERO54 = 0; } enum REQUIRE { ZERO55 = 0; } enum REQUIRE_ONCE { ZERO56 = 0; } enum RETURN { ZERO57 = 0; } +enum SELF { ZERO79 = 0; } enum STATIC { ZERO58 = 0; } enum SWITCH { ZERO59 = 0; } enum THROW { ZERO60 = 0; } diff --git a/php/tests/proto/test_reserved_enum_value_lower.proto b/php/tests/proto/test_reserved_enum_value_lower.proto index 46829060fc..ca5a7c7352 100644 --- a/php/tests/proto/test_reserved_enum_value_lower.proto +++ b/php/tests/proto/test_reserved_enum_value_lower.proto @@ -53,6 +53,7 @@ enum NotAllowed { namespace = 47; new = 48; or = 49; + parent = 77; print = 50; private = 51; protected = 52; @@ -60,6 +61,7 @@ enum NotAllowed { require = 54; require_once = 55; return = 56; + self = 78; static = 57; switch = 58; throw = 59; diff --git a/php/tests/proto/test_reserved_enum_value_upper.proto b/php/tests/proto/test_reserved_enum_value_upper.proto index f30d7ff1ef..6b4040d5e4 100644 --- a/php/tests/proto/test_reserved_enum_value_upper.proto +++ b/php/tests/proto/test_reserved_enum_value_upper.proto @@ -53,6 +53,7 @@ enum NotAllowed { NAMESPACE = 47; NEW = 48; OR = 49; + PARENT = 77; PRINT = 50; PRIVATE = 51; PROTECTED = 52; @@ -60,6 +61,7 @@ enum NotAllowed { REQUIRE = 54; REQUIRE_ONCE = 55; RETURN = 56; + SELF = 78; STATIC = 57; SWITCH = 58; THROW = 59; diff --git a/php/tests/proto/test_reserved_message_lower.proto b/php/tests/proto/test_reserved_message_lower.proto index 05344fff59..2390a87dd6 100644 --- a/php/tests/proto/test_reserved_message_lower.proto +++ b/php/tests/proto/test_reserved_message_lower.proto @@ -52,6 +52,7 @@ message match {} message namespace {} message new {} message or {} +message parent {} message print {} message private {} message protected {} @@ -59,6 +60,7 @@ message public {} message require {} message require_once {} message return {} +message self {} message static {} message switch {} message throw {} diff --git a/php/tests/proto/test_reserved_message_upper.proto b/php/tests/proto/test_reserved_message_upper.proto index 853a17d405..9f55330223 100644 --- a/php/tests/proto/test_reserved_message_upper.proto +++ b/php/tests/proto/test_reserved_message_upper.proto @@ -52,6 +52,7 @@ message MATCH {} message NAMESPACE {} message NEW {} message OR {} +message PARENT {} message PRINT {} message PRIVATE {} message PROTECTED {} @@ -59,6 +60,7 @@ message PUBLIC {} message REQUIRE {} message REQUIRE_ONCE {} message RETURN {} +message SELF {} message STATIC {} message SWITCH {} message THROW {} diff --git a/protobuf.bzl b/protobuf.bzl index 826b6100b8..ffd7c8df67 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -391,6 +391,70 @@ internal_gen_well_known_protos_java = rule( }, ) +def _internal_gen_kt_protos(ctx): + args = ctx.actions.args() + + deps = [d[ProtoInfo] for d in ctx.attr.deps] + + srcjar = ctx.actions.declare_file("{}.srcjar".format(ctx.attr.name)) + if ctx.attr.lite: + out = "lite:%s" % srcjar.path + else: + out = srcjar + + args.add("--kotlin_out", out) + + descriptors = depset( + transitive = [dep.transitive_descriptor_sets for dep in deps], + ) + args.add_joined( + "--descriptor_set_in", + descriptors, + join_with = ctx.configuration.host_path_separator, + ) + + for dep in deps: + if "." == dep.proto_source_root: + args.add_all([src.path for src in dep.direct_sources]) + else: + source_root = dep.proto_source_root + offset = len(source_root) + 1 # + '/'. + args.add_all([src.path[offset:] for src in dep.direct_sources]) + + ctx.actions.run( + executable = ctx.executable._protoc, + inputs = descriptors, + outputs = [srcjar], + arguments = [args], + use_default_shell_env = True, + ) + + return [ + DefaultInfo( + files = depset([srcjar]), + ), + ] + +internal_gen_kt_protos = rule( + implementation = _internal_gen_kt_protos, + attrs = { + "deps": attr.label_list( + mandatory = True, + providers = [ProtoInfo], + ), + "lite": attr.bool( + default = False, + ), + "_protoc": attr.label( + executable = True, + cfg = "exec", + default = "//:protoc", + ), + }, +) + + + def internal_copied_filegroup(name, srcs, strip_prefix, dest, **kwargs): """Macro to copy files to a different directory and then create a filegroup. diff --git a/protobuf_release.bzl b/protobuf_release.bzl new file mode 100644 index 0000000000..cac0a0c52a --- /dev/null +++ b/protobuf_release.bzl @@ -0,0 +1,48 @@ +""" +Generates package naming variables for use with rules_pkg. +""" + +load("@rules_pkg//:providers.bzl", "PackageVariablesInfo") +load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") +load(":protobuf_version.bzl", "PROTOBUF_VERSION") + +def _package_naming_impl(ctx): + values = {} + values["version"] = PROTOBUF_VERSION + + # infer from the current cpp toolchain. + toolchain = find_cpp_toolchain(ctx) + cpu = toolchain.cpu + system_name = toolchain.target_gnu_system_name + + # rename cpus to match what we want artifacts to be + if cpu == "systemz": + cpu = "s390_64" + elif cpu == "aarch64": + cpu = "aarch_64" + + # use the system name to determine the os and then create platform names + if "apple" in system_name: + values["platform"] = "osx-" + cpu + elif "linux" in system_name: + values["platform"] = "linux-" + cpu + elif "mingw" in system_name: + if "cpu" == "x86_64": + values["platform"] = "win64" + else: + values["platform"] = "win32" + else: + values["platform"] = "unknown" + + return PackageVariablesInfo(values = values) + + +package_naming = rule( + implementation = _package_naming_impl, + attrs = { + # Necessary data dependency for find_cpp_toolchain. + "_cc_toolchain": attr.label(default = Label("@bazel_tools//tools/cpp:current_cc_toolchain")), + }, + toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], + incompatible_use_toolchain_transition = True, +) diff --git a/python/google/protobuf/__init__.py b/python/google/protobuf/__init__.py index b8122e82ae..9319c67f1f 100644 --- a/python/google/protobuf/__init__.py +++ b/python/google/protobuf/__init__.py @@ -30,4 +30,4 @@ # Copyright 2007 Google Inc. All Rights Reserved. -__version__ = '3.18.1' +__version__ = '3.19.4' diff --git a/python/google/protobuf/descriptor.py b/python/google/protobuf/descriptor.py index 9615a582fb..ad70be9a11 100644 --- a/python/google/protobuf/descriptor.py +++ b/python/google/protobuf/descriptor.py @@ -667,7 +667,7 @@ class EnumDescriptor(_NestedDescriptorBase): full_name (str): Full name of the type, including package name and any enclosing type(s). - values (list[EnumValueDescriptors]): List of the values + values (list[EnumValueDescriptor]): List of the values in this enum. values_by_name (dict(str, EnumValueDescriptor)): Same as :attr:`values`, but indexed by the "name" field of each EnumValueDescriptor. diff --git a/python/google/protobuf/internal/_parameterized.py b/python/google/protobuf/internal/_parameterized.py index 6b6ba657e6..afdbb78c36 100755 --- a/python/google/protobuf/internal/_parameterized.py +++ b/python/google/protobuf/internal/_parameterized.py @@ -148,10 +148,7 @@ __author__ = 'tmarek@google.com (Torsten Marek)' import functools import re import types -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest import uuid try: diff --git a/python/google/protobuf/internal/api_implementation.cc b/python/google/protobuf/internal/api_implementation.cc index 241942eb3c..33f5b04f49 100644 --- a/python/google/protobuf/internal/api_implementation.cc +++ b/python/google/protobuf/internal/api_implementation.cc @@ -28,6 +28,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#define PY_SSIZE_T_CLEAN #include namespace google { diff --git a/python/google/protobuf/internal/descriptor_database_test.py b/python/google/protobuf/internal/descriptor_database_test.py index ad3ca645f9..3c086b9247 100644 --- a/python/google/protobuf/internal/descriptor_database_test.py +++ b/python/google/protobuf/internal/descriptor_database_test.py @@ -100,7 +100,7 @@ class DescriptorDatabaseTest(unittest.TestCase): self.assertEqual(file_desc_proto2, db.FindFileContainingSymbol( 'protobuf_unittest.TestAllTypes.none_field')) - with self.assertRaisesRegexp(KeyError, r'\'protobuf_unittest\.NoneMessage\''): + with self.assertRaisesRegex(KeyError, r'\'protobuf_unittest\.NoneMessage\''): db.FindFileContainingSymbol('protobuf_unittest.NoneMessage') def testConflictRegister(self): diff --git a/python/google/protobuf/internal/descriptor_pool_test.py b/python/google/protobuf/internal/descriptor_pool_test.py index 501c9a65c4..61cec3f9d3 100644 --- a/python/google/protobuf/internal/descriptor_pool_test.py +++ b/python/google/protobuf/internal/descriptor_pool_test.py @@ -654,11 +654,11 @@ class SecondaryDescriptorFromDescriptorDB(DescriptorPoolTestBase, enum_value.number = 0 self.db.Add(file_proto) - self.assertRaisesRegexp(KeyError, 'SubMessage', - self.pool.FindMessageTypeByName, - 'collector.ErrorMessage') - self.assertRaisesRegexp(KeyError, 'SubMessage', - self.pool.FindFileByName, 'error_file') + self.assertRaisesRegex(KeyError, 'SubMessage', + self.pool.FindMessageTypeByName, + 'collector.ErrorMessage') + self.assertRaisesRegex(KeyError, 'SubMessage', self.pool.FindFileByName, + 'error_file') with self.assertRaises(KeyError) as exc: self.pool.FindFileByName('none_file') self.assertIn(str(exc.exception), ('\'none_file\'', diff --git a/python/google/protobuf/internal/enum_type_wrapper.py b/python/google/protobuf/internal/enum_type_wrapper.py index 9a53999a43..65d2c7a5c0 100644 --- a/python/google/protobuf/internal/enum_type_wrapper.py +++ b/python/google/protobuf/internal/enum_type_wrapper.py @@ -43,6 +43,15 @@ class EnumTypeWrapper(object): DESCRIPTOR = None + # This is a type alias, which mypy typing stubs can type as + # a genericized parameter constrained to an int, allowing subclasses + # to be typed with more constraint in .pyi stubs + # Eg. + # def MyGeneratedEnum(Message): + # ValueType = NewType('ValueType', int) + # def Name(self, number: MyGeneratedEnum.ValueType) -> str + ValueType = int + def __init__(self, enum_type): """Inits EnumTypeWrapper with an EnumDescriptor.""" self._enum_type = enum_type diff --git a/python/google/protobuf/internal/generator_test.py b/python/google/protobuf/internal/generator_test.py index 0c4299a3d1..9883fce31e 100644 --- a/python/google/protobuf/internal/generator_test.py +++ b/python/google/protobuf/internal/generator_test.py @@ -39,10 +39,7 @@ further ensures that we can use Python protocol message objects as we expect. __author__ = 'robinson@google.com (Will Robinson)' -try: - import unittest2 as unittest #PY26 -except ImportError: - import unittest +import unittest from google.protobuf.internal import test_bad_identifiers_pb2 from google.protobuf import unittest_custom_options_pb2 diff --git a/python/google/protobuf/internal/json_format_test.py b/python/google/protobuf/internal/json_format_test.py index 3545563957..d018c3f2e6 100644 --- a/python/google/protobuf/internal/json_format_test.py +++ b/python/google/protobuf/internal/json_format_test.py @@ -100,10 +100,8 @@ class JsonFormatBase(unittest.TestCase): def CheckError(self, text, error_message): message = json_format_proto3_pb2.TestMessage() - self.assertRaisesRegexp( - json_format.ParseError, - error_message, - json_format.Parse, text, message) + self.assertRaisesRegex(json_format.ParseError, error_message, + json_format.Parse, text, message) class JsonFormatTest(JsonFormatBase): @@ -812,9 +810,8 @@ class JsonFormatTest(JsonFormatBase): json_format.Parse('{"messageValue": {}}', parsed_message) self.assertTrue(parsed_message.HasField('message_value')) # Null is not allowed to be used as an element in repeated field. - self.assertRaisesRegexp( - json_format.ParseError, - r'Failed to parse repeatedInt32Value field: ' + self.assertRaisesRegex( + json_format.ParseError, r'Failed to parse repeatedInt32Value field: ' r'null is not allowed to be used as an element in a repeated field ' r'at TestMessage.repeatedInt32Value\[1\].', json_format.Parse, '{"repeatedInt32Value":[1, null]}', parsed_message) @@ -901,7 +898,7 @@ class JsonFormatTest(JsonFormatBase): json_format.Parse(text, message) # Proto2 does not accept unknown enums. message = unittest_pb2.TestAllTypes() - self.assertRaisesRegexp( + self.assertRaisesRegex( json_format.ParseError, 'Failed to parse optionalNestedEnum field: Invalid enum value 12345 ' 'for enum type protobuf_unittest.TestAllTypes.NestedEnum at ' @@ -1019,28 +1016,25 @@ class JsonFormatTest(JsonFormatBase): def testInvalidMap(self): message = json_format_proto3_pb2.TestMap() text = '{"int32Map": {"null": 2, "2": 3}}' - self.assertRaisesRegexp( - json_format.ParseError, - 'Failed to parse int32Map field: invalid literal', - json_format.Parse, text, message) + self.assertRaisesRegex(json_format.ParseError, + 'Failed to parse int32Map field: invalid literal', + json_format.Parse, text, message) text = '{"int32Map": {1: 2, "2": 3}}' - self.assertRaisesRegexp( - json_format.ParseError, - (r'Failed to load JSON: Expecting property name' - r'( enclosed in double quotes)?: line 1'), - json_format.Parse, text, message) + self.assertRaisesRegex(json_format.ParseError, + (r'Failed to load JSON: Expecting property name' + r'( enclosed in double quotes)?: line 1'), + json_format.Parse, text, message) text = '{"boolMap": {"null": 1}}' - self.assertRaisesRegexp( + self.assertRaisesRegex( json_format.ParseError, 'Failed to parse boolMap field: Expected "true" or "false", not null at ' 'TestMap.boolMap.key', json_format.Parse, text, message) text = r'{"stringMap": {"a": 3, "\u0061": 2}}' - self.assertRaisesRegexp( - json_format.ParseError, - 'Failed to load JSON: duplicate key a', - json_format.Parse, text, message) + self.assertRaisesRegex(json_format.ParseError, + 'Failed to load JSON: duplicate key a', + json_format.Parse, text, message) text = r'{"stringMap": 0}' - self.assertRaisesRegexp( + self.assertRaisesRegex( json_format.ParseError, 'Failed to parse stringMap field: Map field string_map must be ' 'in a dict which is 0 at TestMap.stringMap.', json_format.Parse, text, @@ -1055,27 +1049,24 @@ class JsonFormatTest(JsonFormatBase): ' format \'%Y-%m-%dT%H:%M:%S\' at TestTimestamp.value.', json_format.Parse, text, message) text = '{"value": "1970-01-01T00:00:00.0123456789012Z"}' - self.assertRaisesRegexp( + self.assertRaisesRegex( json_format.ParseError, - 'nanos 0123456789012 more than 9 fractional digits.', - json_format.Parse, text, message) + 'nanos 0123456789012 more than 9 fractional digits.', json_format.Parse, + text, message) text = '{"value": "1972-01-01T01:00:00.01+08"}' - self.assertRaisesRegexp( - json_format.ParseError, - (r'Invalid timezone offset value: \+08.'), - json_format.Parse, text, message) + self.assertRaisesRegex(json_format.ParseError, + (r'Invalid timezone offset value: \+08.'), + json_format.Parse, text, message) # Time smaller than minimum time. text = '{"value": "0000-01-01T00:00:00Z"}' - self.assertRaisesRegexp( + self.assertRaisesRegex( json_format.ParseError, 'Failed to parse value field: year (0 )?is out of range.', json_format.Parse, text, message) # Time bigger than maximum time. message.value.seconds = 253402300800 - self.assertRaisesRegexp( - OverflowError, - 'date value out of range', - json_format.MessageToJson, message) + self.assertRaisesRegex(OverflowError, 'date value out of range', + json_format.MessageToJson, message) # Lower case t does not accept. text = '{"value": "0001-01-01t00:00:00Z"}' with self.assertRaises(json_format.ParseError) as e: @@ -1097,11 +1088,10 @@ class JsonFormatTest(JsonFormatBase): def testInvalidListValue(self): message = json_format_proto3_pb2.TestListValue() text = '{"value": 1234}' - self.assertRaisesRegexp( + self.assertRaisesRegex( json_format.ParseError, r'Failed to parse value field: ListValue must be in \[\] which is ' - '1234 at TestListValue.value.', - json_format.Parse, text, message) + '1234 at TestListValue.value.', json_format.Parse, text, message) class UnknownClass(object): @@ -1116,16 +1106,15 @@ class JsonFormatTest(JsonFormatBase): def testInvalidStruct(self): message = json_format_proto3_pb2.TestStruct() text = '{"value": 1234}' - self.assertRaisesRegexp( + self.assertRaisesRegex( json_format.ParseError, 'Failed to parse value field: Struct must be in a dict which is ' - '1234 at TestStruct.value', - json_format.Parse, text, message) + '1234 at TestStruct.value', json_format.Parse, text, message) def testTimestampInvalidStringValue(self): message = json_format_proto3_pb2.TestTimestamp() text = '{"value": {"foo": 123}}' - self.assertRaisesRegexp( + self.assertRaisesRegex( json_format.ParseError, r"Timestamp JSON value not a string: {u?'foo': 123}", json_format.Parse, text, message) @@ -1133,15 +1122,14 @@ class JsonFormatTest(JsonFormatBase): def testDurationInvalidStringValue(self): message = json_format_proto3_pb2.TestDuration() text = '{"value": {"foo": 123}}' - self.assertRaisesRegexp( - json_format.ParseError, - r"Duration JSON value not a string: {u?'foo': 123}", json_format.Parse, - text, message) + self.assertRaisesRegex(json_format.ParseError, + r"Duration JSON value not a string: {u?'foo': 123}", + json_format.Parse, text, message) def testFieldMaskInvalidStringValue(self): message = json_format_proto3_pb2.TestFieldMask() text = '{"value": {"foo": 123}}' - self.assertRaisesRegexp( + self.assertRaisesRegex( json_format.ParseError, r"FieldMask JSON value not a string: {u?'foo': 123}", json_format.Parse, text, message) @@ -1149,10 +1137,7 @@ class JsonFormatTest(JsonFormatBase): def testInvalidAny(self): message = any_pb2.Any() text = '{"@type": "type.googleapis.com/google.protobuf.Int32Value"}' - self.assertRaisesRegexp( - KeyError, - 'value', - json_format.Parse, text, message) + self.assertRaisesRegex(KeyError, 'value', json_format.Parse, text, message) text = '{"value": 1234}' self.assertRaisesRegex(json_format.ParseError, '@type is missing when parsing any message at Any', @@ -1247,12 +1232,10 @@ class JsonFormatTest(JsonFormatBase): def __repr__(self): return 'v' message = json_format_proto3_pb2.TestValue() - self.assertRaisesRegexp( + self.assertRaisesRegex( json_format.ParseError, r"Value v has unexpected type .", - json_format.ParseDict, - {'value': UnknownClass()}, - message) + json_format.ParseDict, {'value': UnknownClass()}, message) def testMessageToDict(self): message = json_format_proto3_pb2.TestMessage() diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py index e8d86a2df8..a5c5a0d740 100644 --- a/python/google/protobuf/internal/message_test.py +++ b/python/google/protobuf/internal/message_test.py @@ -75,21 +75,19 @@ from google.protobuf.internal import _parameterized UCS2_MAXUNICODE = 65535 - warnings.simplefilter('error', DeprecationWarning) -@_parameterized.named_parameters( - ('_proto2', unittest_pb2), - ('_proto3', unittest_proto3_arena_pb2)) +@_parameterized.named_parameters(('_proto2', unittest_pb2), + ('_proto3', unittest_proto3_arena_pb2)) @testing_refleaks.TestCase class MessageTest(unittest.TestCase): def testBadUtf8String(self, message_module): if api_implementation.Type() != 'python': - self.skipTest("Skipping testBadUtf8String, currently only the python " - "api implementation raises UnicodeDecodeError when a " - "string field contains bad utf-8.") + self.skipTest('Skipping testBadUtf8String, currently only the python ' + 'api implementation raises UnicodeDecodeError when a ' + 'string field contains bad utf-8.') bad_utf8_data = test_util.GoldenFileData('bad_utf8_string') with self.assertRaises(UnicodeDecodeError) as context: message_module.TestAllTypes.FromString(bad_utf8_data) @@ -100,8 +98,7 @@ class MessageTest(unittest.TestCase): # and doesn't preserve unknown fields, so for proto3 we use a golden # message that doesn't have these fields set. if message_module is unittest_pb2: - golden_data = test_util.GoldenFileData( - 'golden_message_oneof_implemented') + golden_data = test_util.GoldenFileData('golden_message_oneof_implemented') else: golden_data = test_util.GoldenFileData('golden_message_proto3') @@ -440,8 +437,7 @@ class MessageTest(unittest.TestCase): except TypeError: pass self.assertEqual(2, len(msg.repeated_nested_message)) - self.assertEqual([1, 2], - [m.bb for m in msg.repeated_nested_message]) + self.assertEqual([1, 2], [m.bb for m in msg.repeated_nested_message]) def testInsertRepeatedCompositeField(self, message_module): msg = message_module.TestAllTypes() @@ -463,22 +459,22 @@ class MessageTest(unittest.TestCase): self.assertEqual(5, len(msg.repeated_nested_message)) self.assertEqual([-1000, 2, -1, 1, 3], [m.bb for m in msg.repeated_nested_message]) - self.assertEqual(str(msg), - 'repeated_nested_message {\n' - ' bb: -1000\n' - '}\n' - 'repeated_nested_message {\n' - ' bb: 2\n' - '}\n' - 'repeated_nested_message {\n' - ' bb: -1\n' - '}\n' - 'repeated_nested_message {\n' - ' bb: 1\n' - '}\n' - 'repeated_nested_message {\n' - ' bb: 3\n' - '}\n') + self.assertEqual( + str(msg), 'repeated_nested_message {\n' + ' bb: -1000\n' + '}\n' + 'repeated_nested_message {\n' + ' bb: 2\n' + '}\n' + 'repeated_nested_message {\n' + ' bb: -1\n' + '}\n' + 'repeated_nested_message {\n' + ' bb: 1\n' + '}\n' + 'repeated_nested_message {\n' + ' bb: 3\n' + '}\n') self.assertEqual(sub_msg.bb, 1) def testMergeFromRepeatedField(self, message_module): @@ -497,8 +493,7 @@ class MessageTest(unittest.TestCase): self.assertEqual(4, len(msg.repeated_int32)) msg.repeated_nested_message.MergeFrom(other_msg.repeated_nested_message) - self.assertEqual([1, 2, 3, 4], - [m.bb for m in msg.repeated_nested_message]) + self.assertEqual([1, 2, 3, 4], [m.bb for m in msg.repeated_nested_message]) def testAddWrongRepeatedNestedField(self, message_module): msg = message_module.TestAllTypes() @@ -543,8 +538,7 @@ class MessageTest(unittest.TestCase): msg.repeated_nested_message.add(bb=3) msg.repeated_nested_message.add(bb=4) - self.assertEqual([1, 2, 3, 4], - [m.bb for m in msg.repeated_nested_message]) + self.assertEqual([1, 2, 3, 4], [m.bb for m in msg.repeated_nested_message]) self.assertEqual([4, 3, 2, 1], [m.bb for m in reversed(msg.repeated_nested_message)]) self.assertEqual([4, 3, 2, 1], @@ -627,8 +621,9 @@ class MessageTest(unittest.TestCase): self.assertEqual(message.repeated_nested_message[3].bb, 4) self.assertEqual(message.repeated_nested_message[4].bb, 5) self.assertEqual(message.repeated_nested_message[5].bb, 6) - self.assertEqual(str(message.repeated_nested_message), - '[bb: 1\n, bb: 2\n, bb: 3\n, bb: 4\n, bb: 5\n, bb: 6\n]') + self.assertEqual( + str(message.repeated_nested_message), + '[bb: 1\n, bb: 2\n, bb: 3\n, bb: 4\n, bb: 5\n, bb: 6\n]') def testSortingRepeatedCompositeFieldsStable(self, message_module): """Check passing a custom comparator to sort a repeated composite field.""" @@ -642,18 +637,16 @@ class MessageTest(unittest.TestCase): message.repeated_nested_message.add().bb = 24 message.repeated_nested_message.add().bb = 10 message.repeated_nested_message.sort(key=lambda z: z.bb // 10) - self.assertEqual( - [13, 11, 10, 21, 20, 24, 33], - [n.bb for n in message.repeated_nested_message]) + self.assertEqual([13, 11, 10, 21, 20, 24, 33], + [n.bb for n in message.repeated_nested_message]) # Make sure that for the C++ implementation, the underlying fields # are actually reordered. pb = message.SerializeToString() message.Clear() message.MergeFromString(pb) - self.assertEqual( - [13, 11, 10, 21, 20, 24, 33], - [n.bb for n in message.repeated_nested_message]) + self.assertEqual([13, 11, 10, 21, 20, 24, 33], + [n.bb for n in message.repeated_nested_message]) def testRepeatedCompositeFieldSortArguments(self, message_module): """Check sorting a repeated composite field using list.sort() arguments.""" @@ -826,7 +819,7 @@ class MessageTest(unittest.TestCase): self.assertTrue(m.HasField('oneof_uint32')) self.assertFalse(m.HasField('oneof_string')) - m.oneof_string = "" + m.oneof_string = '' self.assertEqual('oneof_string', m.WhichOneof('oneof_field')) self.assertTrue(m.HasField('oneof_string')) self.assertFalse(m.HasField('oneof_uint32')) @@ -973,7 +966,9 @@ class MessageTest(unittest.TestCase): def testAssignByteStringToUnicodeField(self, message_module): """Assigning a byte string to a string field should result - in the value being converted to a Unicode string.""" + + in the value being converted to a Unicode string. + """ m = message_module.TestAllTypes() m.optional_string = str('') self.assertIsInstance(m.optional_string, str) @@ -1001,8 +996,7 @@ class MessageTest(unittest.TestCase): with self.assertRaises(NameError) as _: m.repeated_int32.extend(a for i in range(10)) # pylint: disable=undefined-variable with self.assertRaises(NameError) as _: - m.repeated_nested_enum.extend( - a for i in range(10)) # pylint: disable=undefined-variable + m.repeated_nested_enum.extend(a for i in range(10)) # pylint: disable=undefined-variable FALSY_VALUES = [None, False, 0, 0.0, b'', u'', bytearray(), [], {}, set()] @@ -1179,14 +1173,12 @@ class MessageTest(unittest.TestCase): pickle.dumps(m.repeated_int32, pickle.HIGHEST_PROTOCOL) def testSortEmptyRepeatedCompositeContainer(self, message_module): - """Exercise a scenario that has led to segfaults in the past. - """ + """Exercise a scenario that has led to segfaults in the past.""" m = message_module.TestAllTypes() m.repeated_nested_message.sort() def testHasFieldOnRepeatedField(self, message_module): - """Using HasField on a repeated field should raise an exception. - """ + """Using HasField on a repeated field should raise an exception.""" m = message_module.TestAllTypes() with self.assertRaises(ValueError) as _: m.HasField('repeated_int32') @@ -1226,6 +1218,7 @@ class MessageTest(unittest.TestCase): def testReleasedNestedMessages(self, message_module): """A case that lead to a segfault when a message detached from its parent + container has itself a child container. """ m = message_module.NestedTestAllTypes() @@ -1271,17 +1264,17 @@ class Proto2Test(unittest.TestCase): def testFieldPresence(self): message = unittest_pb2.TestAllTypes() - self.assertFalse(message.HasField("optional_int32")) - self.assertFalse(message.HasField("optional_bool")) - self.assertFalse(message.HasField("optional_nested_message")) + self.assertFalse(message.HasField('optional_int32')) + self.assertFalse(message.HasField('optional_bool')) + self.assertFalse(message.HasField('optional_nested_message')) with self.assertRaises(ValueError): - message.HasField("field_doesnt_exist") + message.HasField('field_doesnt_exist') with self.assertRaises(ValueError): - message.HasField("repeated_int32") + message.HasField('repeated_int32') with self.assertRaises(ValueError): - message.HasField("repeated_nested_message") + message.HasField('repeated_nested_message') self.assertEqual(0, message.optional_int32) self.assertEqual(False, message.optional_bool) @@ -1291,27 +1284,27 @@ class Proto2Test(unittest.TestCase): message.optional_int32 = 0 message.optional_bool = False message.optional_nested_message.bb = 0 - self.assertTrue(message.HasField("optional_int32")) - self.assertTrue(message.HasField("optional_bool")) - self.assertTrue(message.HasField("optional_nested_message")) + self.assertTrue(message.HasField('optional_int32')) + self.assertTrue(message.HasField('optional_bool')) + self.assertTrue(message.HasField('optional_nested_message')) # Set the fields to non-default values. message.optional_int32 = 5 message.optional_bool = True message.optional_nested_message.bb = 15 - self.assertTrue(message.HasField(u"optional_int32")) - self.assertTrue(message.HasField("optional_bool")) - self.assertTrue(message.HasField("optional_nested_message")) + self.assertTrue(message.HasField(u'optional_int32')) + self.assertTrue(message.HasField('optional_bool')) + self.assertTrue(message.HasField('optional_nested_message')) # Clearing the fields unsets them and resets their value to default. - message.ClearField("optional_int32") - message.ClearField(u"optional_bool") - message.ClearField("optional_nested_message") + message.ClearField('optional_int32') + message.ClearField(u'optional_bool') + message.ClearField('optional_nested_message') - self.assertFalse(message.HasField("optional_int32")) - self.assertFalse(message.HasField("optional_bool")) - self.assertFalse(message.HasField("optional_nested_message")) + self.assertFalse(message.HasField('optional_int32')) + self.assertFalse(message.HasField('optional_bool')) + self.assertFalse(message.HasField('optional_nested_message')) self.assertEqual(0, message.optional_int32) self.assertEqual(False, message.optional_bool) self.assertEqual(0, message.optional_nested_message.bb) @@ -1361,16 +1354,17 @@ class Proto2Test(unittest.TestCase): msg1 = more_extensions_pb2.TopLevelMessage() msg2 = more_extensions_pb2.TopLevelMessage() # Cpp extension will lazily create a sub message which is immutable. - self.assertEqual(0, msg1.submessage.Extensions[ - more_extensions_pb2.optional_int_extension]) + self.assertEqual( + 0, + msg1.submessage.Extensions[more_extensions_pb2.optional_int_extension]) self.assertFalse(msg1.HasField('submessage')) - msg2.submessage.Extensions[ - more_extensions_pb2.optional_int_extension] = 123 + msg2.submessage.Extensions[more_extensions_pb2.optional_int_extension] = 123 # Make sure cmessage and extensions pointing to a mutable message # after merge instead of the lazily created message. msg1.MergeFrom(msg2) - self.assertEqual(123, msg1.submessage.Extensions[ - more_extensions_pb2.optional_int_extension]) + self.assertEqual( + 123, + msg1.submessage.Extensions[more_extensions_pb2.optional_int_extension]) def testGoldenExtensions(self): golden_data = test_util.GoldenFileData('golden_message') @@ -1404,17 +1398,19 @@ class Proto2Test(unittest.TestCase): # This is still an incomplete proto - so serializing should fail self.assertRaises(message.EncodeError, unpickled_message.SerializeToString) - # TODO(haberman): this isn't really a proto2-specific test except that this # message has a required field in it. Should probably be factored out so # that we can test the other parts with proto3. def testParsingMerge(self): """Check the merge behavior when a required or optional field appears - multiple times in the input.""" + + multiple times in the input. + """ messages = [ unittest_pb2.TestAllTypes(), unittest_pb2.TestAllTypes(), - unittest_pb2.TestAllTypes() ] + unittest_pb2.TestAllTypes() + ] messages[0].optional_int32 = 1 messages[1].optional_int64 = 2 messages[2].optional_int32 = 3 @@ -1447,15 +1443,16 @@ class Proto2Test(unittest.TestCase): self.assertEqual(parsing_merge.optional_all_types, merged_message) self.assertEqual(parsing_merge.optionalgroup.optional_group_all_types, merged_message) - self.assertEqual(parsing_merge.Extensions[ - unittest_pb2.TestParsingMerge.optional_ext], - merged_message) + self.assertEqual( + parsing_merge.Extensions[unittest_pb2.TestParsingMerge.optional_ext], + merged_message) # Repeated fields should not be merged. self.assertEqual(len(parsing_merge.repeated_all_types), 3) self.assertEqual(len(parsing_merge.repeatedgroup), 3) - self.assertEqual(len(parsing_merge.Extensions[ - unittest_pb2.TestParsingMerge.repeated_ext]), 3) + self.assertEqual( + len(parsing_merge.Extensions[ + unittest_pb2.TestParsingMerge.repeated_ext]), 3) def testPythonicInit(self): message = unittest_pb2.TestAllTypes( @@ -1467,8 +1464,11 @@ class Proto2Test(unittest.TestCase): optional_nested_message={'bb': 500}, optional_foreign_message={}, optional_nested_enum='BAZ', - repeatedgroup=[{'a': 600}, - {'a': 700}], + repeatedgroup=[{ + 'a': 600 + }, { + 'a': 700 + }], repeated_nested_enum=['FOO', unittest_pb2.TestAllTypes.BAR], default_int32=800, oneof_string='y') @@ -1848,8 +1848,7 @@ class Proto3Test(unittest.TestCase): self.assertEqual(True, msg2.map_bool_bool[True]) self.assertEqual(2, msg2.map_int32_enum[888]) self.assertEqual(456, msg2.map_int32_enum[123]) - self.assertEqual('{-123: -456}', - str(msg2.map_int32_int32)) + self.assertEqual('{-123: -456}', str(msg2.map_int32_int32)) def testMapEntryAlwaysSerialized(self): msg = map_unittest_pb2.TestMap() @@ -1912,8 +1911,9 @@ class Proto3Test(unittest.TestCase): self.assertEqual(2, len(msg2.map_int32_foreign_message)) msg2.map_int32_foreign_message[123].c = 1 # TODO(jieluo): Fix text format for message map. - self.assertIn(str(msg2.map_int32_foreign_message), - ('{-456: , 123: c: 1\n}', '{123: c: 1\n, -456: }')) + self.assertIn( + str(msg2.map_int32_foreign_message), + ('{-456: , 123: c: 1\n}', '{123: c: 1\n, -456: }')) def testNestedMessageMapItemDelete(self): msg = map_unittest_pb2.TestMap() @@ -2041,8 +2041,7 @@ class Proto3Test(unittest.TestCase): # Test when cpp extension cache a map. m1 = map_unittest_pb2.TestMap() m2 = map_unittest_pb2.TestMap() - self.assertEqual(m1.map_int32_foreign_message, - m1.map_int32_foreign_message) + self.assertEqual(m1.map_int32_foreign_message, m1.map_int32_foreign_message) m2.map_int32_foreign_message[123].c = 10 m1.MergeFrom(m2) self.assertEqual(10, m2.map_int32_foreign_message[123].c) @@ -2070,7 +2069,7 @@ class Proto3Test(unittest.TestCase): def testMergeFromBadType(self): msg = map_unittest_pb2.TestMap() - with self.assertRaisesRegexp( + with self.assertRaisesRegex( TypeError, r'Parameter to MergeFrom\(\) must be instance of same class: expected ' r'.+TestMap got int\.'): @@ -2078,7 +2077,7 @@ class Proto3Test(unittest.TestCase): def testCopyFromBadType(self): msg = map_unittest_pb2.TestMap() - with self.assertRaisesRegexp( + with self.assertRaisesRegex( TypeError, r'Parameter to [A-Za-z]*From\(\) must be instance of same class: ' r'expected .+TestMap got int\.'): @@ -2167,6 +2166,34 @@ class Proto3Test(unittest.TestCase): for key in int32_foreign_iter: pass + def testModifyMapEntryWhileIterating(self): + msg = map_unittest_pb2.TestMap() + + msg.map_string_string['abc'] = '123' + msg.map_string_string['def'] = '456' + msg.map_string_string['ghi'] = '789' + + msg.map_int32_foreign_message[5].c = 5 + msg.map_int32_foreign_message[6].c = 6 + msg.map_int32_foreign_message[7].c = 7 + + string_string_keys = list(msg.map_string_string.keys()) + int32_foreign_keys = list(msg.map_int32_foreign_message.keys()) + + keys = [] + for key in msg.map_string_string: + keys.append(key) + msg.map_string_string[key] = '000' + self.assertEqual(keys, string_string_keys) + self.assertEqual(keys, list(msg.map_string_string.keys())) + + keys = [] + for key in msg.map_int32_foreign_message: + keys.append(key) + msg.map_int32_foreign_message[key].c = 0 + self.assertEqual(keys, int32_foreign_keys) + self.assertEqual(keys, list(msg.map_int32_foreign_message.keys())) + def testSubmessageMap(self): msg = map_unittest_pb2.TestMap() @@ -2278,7 +2305,7 @@ class Proto3Test(unittest.TestCase): msg1 = map_unittest_pb2.TestMap() msg1.map_string_foreign_message['test'].c = 42 msg2 = map_unittest_pb2.TestMap( - map_string_foreign_message=msg1.map_string_foreign_message) + map_string_foreign_message=msg1.map_string_foreign_message) self.assertEqual(42, msg2.map_string_foreign_message['test'].c) def testMapFieldRaisesCorrectError(self): @@ -2413,24 +2440,21 @@ class Proto3Test(unittest.TestCase): msg2.MergeFromString(serialized) self.assertEqual(msg2.optional_string, u'😍') - msg = unittest_proto3_arena_pb2.TestAllTypes( - optional_string=u'\ud001') + msg = unittest_proto3_arena_pb2.TestAllTypes(optional_string=u'\ud001') self.assertEqual(msg.optional_string, u'\ud001') def testSurrogatesInPython3(self): # Surrogates are rejected at setters in Python3. with self.assertRaises(ValueError): - unittest_proto3_arena_pb2.TestAllTypes( - optional_string=u'\ud801\udc01') + unittest_proto3_arena_pb2.TestAllTypes(optional_string=u'\ud801\udc01') with self.assertRaises(ValueError): - unittest_proto3_arena_pb2.TestAllTypes( - optional_string=b'\xed\xa0\x81') + unittest_proto3_arena_pb2.TestAllTypes(optional_string=b'\xed\xa0\x81') with self.assertRaises(ValueError): - unittest_proto3_arena_pb2.TestAllTypes( - optional_string=u'\ud801') + unittest_proto3_arena_pb2.TestAllTypes(optional_string=u'\ud801') with self.assertRaises(ValueError): - unittest_proto3_arena_pb2.TestAllTypes( - optional_string=u'\ud801\ud801') + unittest_proto3_arena_pb2.TestAllTypes(optional_string=u'\ud801\ud801') + + @testing_refleaks.TestCase @@ -2441,8 +2465,9 @@ class ValidTypeNamesTest(unittest.TestCase): tp_name = str(type(msg)).split("'")[1] valid_names = ('Repeated%sContainer' % base_name, 'Repeated%sFieldContainer' % base_name) - self.assertTrue(any(tp_name.endswith(v) for v in valid_names), - '%r does end with any of %r' % (tp_name, valid_names)) + self.assertTrue( + any(tp_name.endswith(v) for v in valid_names), + '%r does end with any of %r' % (tp_name, valid_names)) parts = tp_name.split('.') class_name = parts[-1] @@ -2455,6 +2480,7 @@ class ValidTypeNamesTest(unittest.TestCase): self.assertImportFromName(pb.repeated_int32, 'Scalar') self.assertImportFromName(pb.repeated_nested_message, 'Composite') + @testing_refleaks.TestCase class PackedFieldTest(unittest.TestCase): @@ -2574,5 +2600,6 @@ class OversizeProtosTest(unittest.TestCase): q.ParseFromString(self.p_serialized) self.assertEqual(self.p.field.payload, q.field.payload) + if __name__ == '__main__': unittest.main() diff --git a/python/google/protobuf/internal/proto_builder_test.py b/python/google/protobuf/internal/proto_builder_test.py index 8e2afaf98c..48077b0a4b 100644 --- a/python/google/protobuf/internal/proto_builder_test.py +++ b/python/google/protobuf/internal/proto_builder_test.py @@ -31,10 +31,7 @@ """Tests for google.protobuf.proto_builder.""" import collections -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest from google.protobuf import descriptor_pb2 # pylint: disable=g-import-not-at-top from google.protobuf import descriptor diff --git a/python/google/protobuf/internal/reflection_test.py b/python/google/protobuf/internal/reflection_test.py index 9343389ed5..66dd0c710a 100644 --- a/python/google/protobuf/internal/reflection_test.py +++ b/python/google/protobuf/internal/reflection_test.py @@ -37,6 +37,7 @@ import copy import gc import operator import struct +import sys import warnings import unittest @@ -376,7 +377,8 @@ class ReflectionTest(unittest.TestCase): self.assertRaises(TypeError, setattr, proto, 'optional_float', 'foo') self.assertRaises(TypeError, setattr, proto, 'optional_double', 'foo') # TODO(jieluo): Fix type checking difference for python and c extension - if api_implementation.Type() == 'python': + if (api_implementation.Type() == 'python' or + (sys.version_info.major, sys.version_info.minor) >= (3, 10)): self.assertRaises(TypeError, setattr, proto, 'optional_bool', 1.1) else: proto.optional_bool = 1.1 @@ -427,7 +429,7 @@ class ReflectionTest(unittest.TestCase): pb.optional_uint64 = '2' # The exact error should propagate with a poorly written custom integer. - with self.assertRaisesRegexp(RuntimeError, 'my_error'): + with self.assertRaisesRegex(RuntimeError, 'my_error'): pb.optional_uint64 = test_util.NonStandardInteger(5, 'my_error') def assetIntegerBoundsChecking(self, integer_fn, message_module): diff --git a/python/google/protobuf/internal/well_known_types_test.py b/python/google/protobuf/internal/well_known_types_test.py index 37aec7e7bf..391290194f 100644 --- a/python/google/protobuf/internal/well_known_types_test.py +++ b/python/google/protobuf/internal/well_known_types_test.py @@ -36,8 +36,6 @@ import collections.abc as collections_abc import datetime import unittest -import dateutil.tz - from google.protobuf import any_pb2 from google.protobuf import duration_pb2 from google.protobuf import field_mask_pb2 @@ -50,9 +48,17 @@ from google.protobuf.internal import test_util from google.protobuf.internal import well_known_types from google.protobuf import descriptor from google.protobuf import text_format -from google3.pyglib import datelib from google.protobuf.internal import _parameterized +try: + # New module in Python 3.9: + import zoneinfo # pylint:disable=g-import-not-at-top + _TZ_JAPAN = zoneinfo.ZoneInfo('Japan') + _TZ_PACIFIC = zoneinfo.ZoneInfo('US/Pacific') +except ImportError: + _TZ_JAPAN = datetime.timezone(datetime.timedelta(hours=9), 'Japan') + _TZ_PACIFIC = datetime.timezone(datetime.timedelta(hours=-8), 'US/Pacific') + class TimeUtilTestBase(_parameterized.TestCase): @@ -270,12 +276,12 @@ class TimeUtilTest(TimeUtilTestBase): # Two hours after the Unix Epoch, around the world. @_parameterized.named_parameters( - ('London', [1970, 1, 1, 2], dateutil.tz.UTC), - ('Tokyo', [1970, 1, 1, 11], dateutil.tz.gettz('Japan')), - ('LA', [1969, 12, 31, 18], dateutil.tz.gettz('US/Pacific')), + ('London', [1970, 1, 1, 2], datetime.timezone.utc), + ('Tokyo', [1970, 1, 1, 11], _TZ_JAPAN), + ('LA', [1969, 12, 31, 18], _TZ_PACIFIC), ) def testTimezoneAwareDatetimeConversion(self, date_parts, tzinfo): - original_datetime = datelib.CreateDatetime(*date_parts, tzinfo=tzinfo) + original_datetime = datetime.datetime(*date_parts, tzinfo=tzinfo) # pylint:disable=g-tzinfo-datetime message = timestamp_pb2.Timestamp() message.FromDatetime(original_datetime) @@ -296,7 +302,7 @@ class TimeUtilTest(TimeUtilTestBase): aware_datetime = message.ToDatetime(tzinfo=tzinfo) self.assertEqual(original_datetime, aware_datetime) self.assertEqual( - datelib.CreateDatetime(1970, 1, 1, 2, tzinfo=dateutil.tz.UTC), + datetime.datetime(1970, 1, 1, 2, tzinfo=datetime.timezone.utc), aware_datetime) self.assertEqual(tzinfo, aware_datetime.tzinfo) @@ -324,85 +330,64 @@ class TimeUtilTest(TimeUtilTestBase): def testInvalidTimestamp(self): message = timestamp_pb2.Timestamp() - self.assertRaisesRegexp( - ValueError, - 'Failed to parse timestamp: missing valid timezone offset.', - message.FromJsonString, - '') - self.assertRaisesRegexp( - ValueError, - 'Failed to parse timestamp: invalid trailing data ' - '1970-01-01T00:00:01Ztrail.', - message.FromJsonString, + self.assertRaisesRegex( + ValueError, 'Failed to parse timestamp: missing valid timezone offset.', + message.FromJsonString, '') + self.assertRaisesRegex( + ValueError, 'Failed to parse timestamp: invalid trailing data ' + '1970-01-01T00:00:01Ztrail.', message.FromJsonString, '1970-01-01T00:00:01Ztrail') - self.assertRaisesRegexp( - ValueError, - 'time data \'10000-01-01T00:00:00\' does not match' - ' format \'%Y-%m-%dT%H:%M:%S\'', - message.FromJsonString, '10000-01-01T00:00:00.00Z') - self.assertRaisesRegexp( - ValueError, - 'nanos 0123456789012 more than 9 fractional digits.', - message.FromJsonString, - '1970-01-01T00:00:00.0123456789012Z') - self.assertRaisesRegexp( + self.assertRaisesRegex( + ValueError, 'time data \'10000-01-01T00:00:00\' does not match' + ' format \'%Y-%m-%dT%H:%M:%S\'', message.FromJsonString, + '10000-01-01T00:00:00.00Z') + self.assertRaisesRegex( + ValueError, 'nanos 0123456789012 more than 9 fractional digits.', + message.FromJsonString, '1970-01-01T00:00:00.0123456789012Z') + self.assertRaisesRegex( ValueError, (r'Invalid timezone offset value: \+08.'), message.FromJsonString, - '1972-01-01T01:00:00.01+08',) - self.assertRaisesRegexp( - ValueError, - 'year (0 )?is out of range', - message.FromJsonString, - '0000-01-01T00:00:00Z') + '1972-01-01T01:00:00.01+08', + ) + self.assertRaisesRegex(ValueError, 'year (0 )?is out of range', + message.FromJsonString, '0000-01-01T00:00:00Z') message.seconds = 253402300800 - self.assertRaisesRegexp( - OverflowError, - 'date value out of range', - message.ToJsonString) + self.assertRaisesRegex(OverflowError, 'date value out of range', + message.ToJsonString) def testInvalidDuration(self): message = duration_pb2.Duration() - self.assertRaisesRegexp( - ValueError, - 'Duration must end with letter "s": 1.', - message.FromJsonString, '1') - self.assertRaisesRegexp( - ValueError, - 'Couldn\'t parse duration: 1...2s.', - message.FromJsonString, '1...2s') + self.assertRaisesRegex(ValueError, 'Duration must end with letter "s": 1.', + message.FromJsonString, '1') + self.assertRaisesRegex(ValueError, 'Couldn\'t parse duration: 1...2s.', + message.FromJsonString, '1...2s') text = '-315576000001.000000000s' - self.assertRaisesRegexp( + self.assertRaisesRegex( ValueError, r'Duration is not valid\: Seconds -315576000001 must be in range' - r' \[-315576000000\, 315576000000\].', - message.FromJsonString, text) + r' \[-315576000000\, 315576000000\].', message.FromJsonString, text) text = '315576000001.000000000s' - self.assertRaisesRegexp( + self.assertRaisesRegex( ValueError, r'Duration is not valid\: Seconds 315576000001 must be in range' - r' \[-315576000000\, 315576000000\].', - message.FromJsonString, text) + r' \[-315576000000\, 315576000000\].', message.FromJsonString, text) message.seconds = -315576000001 message.nanos = 0 - self.assertRaisesRegexp( + self.assertRaisesRegex( ValueError, r'Duration is not valid\: Seconds -315576000001 must be in range' - r' \[-315576000000\, 315576000000\].', - message.ToJsonString) + r' \[-315576000000\, 315576000000\].', message.ToJsonString) message.seconds = 0 message.nanos = 999999999 + 1 - self.assertRaisesRegexp( - ValueError, - r'Duration is not valid\: Nanos 1000000000 must be in range' - r' \[-999999999\, 999999999\].', - message.ToJsonString) + self.assertRaisesRegex( + ValueError, r'Duration is not valid\: Nanos 1000000000 must be in range' + r' \[-999999999\, 999999999\].', message.ToJsonString) message.seconds = -1 message.nanos = 1 - self.assertRaisesRegexp( - ValueError, - r'Duration is not valid\: Sign mismatch.', - message.ToJsonString) + self.assertRaisesRegex(ValueError, + r'Duration is not valid\: Sign mismatch.', + message.ToJsonString) class FieldMaskTest(unittest.TestCase): @@ -724,34 +709,29 @@ class FieldMaskTest(unittest.TestCase): well_known_types._SnakeCaseToCamelCase('foo3_bar')) # No uppercase letter is allowed. - self.assertRaisesRegexp( + self.assertRaisesRegex( ValueError, 'Fail to print FieldMask to Json string: Path name Foo must ' 'not contain uppercase letters.', - well_known_types._SnakeCaseToCamelCase, - 'Foo') + well_known_types._SnakeCaseToCamelCase, 'Foo') # Any character after a "_" must be a lowercase letter. # 1. "_" cannot be followed by another "_". # 2. "_" cannot be followed by a digit. # 3. "_" cannot appear as the last character. - self.assertRaisesRegexp( + self.assertRaisesRegex( ValueError, 'Fail to print FieldMask to Json string: The character after a ' '"_" must be a lowercase letter in path name foo__bar.', - well_known_types._SnakeCaseToCamelCase, - 'foo__bar') - self.assertRaisesRegexp( + well_known_types._SnakeCaseToCamelCase, 'foo__bar') + self.assertRaisesRegex( ValueError, 'Fail to print FieldMask to Json string: The character after a ' '"_" must be a lowercase letter in path name foo_3bar.', - well_known_types._SnakeCaseToCamelCase, - 'foo_3bar') - self.assertRaisesRegexp( + well_known_types._SnakeCaseToCamelCase, 'foo_3bar') + self.assertRaisesRegex( ValueError, 'Fail to print FieldMask to Json string: Trailing "_" in path ' - 'name foo_bar_.', - well_known_types._SnakeCaseToCamelCase, - 'foo_bar_') + 'name foo_bar_.', well_known_types._SnakeCaseToCamelCase, 'foo_bar_') def testCamelCaseToSnakeCase(self): self.assertEqual('foo_bar', @@ -760,11 +740,10 @@ class FieldMaskTest(unittest.TestCase): well_known_types._CamelCaseToSnakeCase('FooBar')) self.assertEqual('foo3_bar', well_known_types._CamelCaseToSnakeCase('foo3Bar')) - self.assertRaisesRegexp( + self.assertRaisesRegex( ValueError, 'Fail to parse FieldMask: Path name foo_bar must not contain "_"s.', - well_known_types._CamelCaseToSnakeCase, - 'foo_bar') + well_known_types._CamelCaseToSnakeCase, 'foo_bar') class StructTest(unittest.TestCase): diff --git a/python/google/protobuf/message.py b/python/google/protobuf/message.py index ee46d0e4c9..76c6802f70 100644 --- a/python/google/protobuf/message.py +++ b/python/google/protobuf/message.py @@ -194,6 +194,9 @@ class Message(object): """Parse serialized protocol buffer data into this message. Like :func:`MergeFromString()`, except we clear the object first. + + Raises: + message.DecodeError if the input cannot be parsed. """ self.Clear() return self.MergeFromString(serialized) diff --git a/python/google/protobuf/proto_api.h b/python/google/protobuf/proto_api.h index 2e2156a56e..9969a91f44 100644 --- a/python/google/protobuf/proto_api.h +++ b/python/google/protobuf/proto_api.h @@ -45,6 +45,7 @@ #ifndef GOOGLE_PROTOBUF_PYTHON_PROTO_API_H__ #define GOOGLE_PROTOBUF_PYTHON_PROTO_API_H__ +#define PY_SSIZE_T_CLEAN #include #include diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc index cbf6887200..162531226e 100644 --- a/python/google/protobuf/pyext/descriptor.cc +++ b/python/google/protobuf/pyext/descriptor.cc @@ -32,6 +32,7 @@ #include +#define PY_SSIZE_T_CLEAN #include #include @@ -415,11 +416,15 @@ static PyGetSetDef Getters[] = { PyTypeObject PyBaseDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME - ".DescriptorBase", // tp_name - sizeof(PyBaseDescriptor), // tp_basicsize - 0, // tp_itemsize - (destructor)Dealloc, // tp_dealloc - 0, // tp_print + ".DescriptorBase", // tp_name + sizeof(PyBaseDescriptor), // tp_basicsize + 0, // tp_itemsize + (destructor)Dealloc, // tp_dealloc +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif nullptr, // tp_getattr nullptr, // tp_setattr nullptr, // tp_compare @@ -685,11 +690,15 @@ static PyMethodDef Methods[] = { PyTypeObject PyMessageDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME - ".MessageDescriptor", // tp_name - sizeof(PyBaseDescriptor), // tp_basicsize - 0, // tp_itemsize - nullptr, // tp_dealloc - 0, // tp_print + ".MessageDescriptor", // tp_name + sizeof(PyBaseDescriptor), // tp_basicsize + 0, // tp_itemsize + nullptr, // tp_dealloc +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif nullptr, // tp_getattr nullptr, // tp_setattr nullptr, // tp_compare @@ -1011,11 +1020,15 @@ static PyMethodDef Methods[] = { PyTypeObject PyFieldDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME - ".FieldDescriptor", // tp_name - sizeof(PyBaseDescriptor), // tp_basicsize - 0, // tp_itemsize - nullptr, // tp_dealloc - 0, // tp_print + ".FieldDescriptor", // tp_name + sizeof(PyBaseDescriptor), // tp_basicsize + 0, // tp_itemsize + nullptr, // tp_dealloc +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif nullptr, // tp_getattr nullptr, // tp_setattr nullptr, // tp_compare @@ -1167,11 +1180,15 @@ static PyGetSetDef Getters[] = { PyTypeObject PyEnumDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME - ".EnumDescriptor", // tp_name - sizeof(PyBaseDescriptor), // tp_basicsize - 0, // tp_itemsize - nullptr, // tp_dealloc - 0, // tp_print + ".EnumDescriptor", // tp_name + sizeof(PyBaseDescriptor), // tp_basicsize + 0, // tp_itemsize + nullptr, // tp_dealloc +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif nullptr, // tp_getattr nullptr, // tp_setattr nullptr, // tp_compare @@ -1288,11 +1305,15 @@ static PyMethodDef Methods[] = { PyTypeObject PyEnumValueDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME - ".EnumValueDescriptor", // tp_name - sizeof(PyBaseDescriptor), // tp_basicsize - 0, // tp_itemsize - nullptr, // tp_dealloc - 0, // tp_print + ".EnumValueDescriptor", // tp_name + sizeof(PyBaseDescriptor), // tp_basicsize + 0, // tp_itemsize + nullptr, // tp_dealloc +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif nullptr, // tp_getattr nullptr, // tp_setattr nullptr, // tp_compare @@ -1477,40 +1498,44 @@ PyTypeObject PyFileDescriptor_Type = { sizeof(PyFileDescriptor), // tp_basicsize 0, // tp_itemsize (destructor)file_descriptor::Dealloc, // tp_dealloc - 0, // tp_print - nullptr, // tp_getattr - nullptr, // tp_setattr - nullptr, // tp_compare - nullptr, // tp_repr - nullptr, // tp_as_number - nullptr, // tp_as_sequence - nullptr, // tp_as_mapping - nullptr, // tp_hash - nullptr, // tp_call - nullptr, // tp_str - nullptr, // tp_getattro - nullptr, // tp_setattro - nullptr, // tp_as_buffer - Py_TPFLAGS_DEFAULT, // tp_flags - "A File Descriptor", // tp_doc - nullptr, // tp_traverse - nullptr, // tp_clear - nullptr, // tp_richcompare - 0, // tp_weaklistoffset - nullptr, // tp_iter - nullptr, // tp_iternext - file_descriptor::Methods, // tp_methods - nullptr, // tp_members - file_descriptor::Getters, // tp_getset - &descriptor::PyBaseDescriptor_Type, // tp_base - nullptr, // tp_dict - nullptr, // tp_descr_get - nullptr, // tp_descr_set - 0, // tp_dictoffset - nullptr, // tp_init - nullptr, // tp_alloc - nullptr, // tp_new - PyObject_GC_Del, // tp_free +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif + nullptr, // tp_getattr + nullptr, // tp_setattr + nullptr, // tp_compare + nullptr, // tp_repr + nullptr, // tp_as_number + nullptr, // tp_as_sequence + nullptr, // tp_as_mapping + nullptr, // tp_hash + nullptr, // tp_call + nullptr, // tp_str + nullptr, // tp_getattro + nullptr, // tp_setattro + nullptr, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "A File Descriptor", // tp_doc + nullptr, // tp_traverse + nullptr, // tp_clear + nullptr, // tp_richcompare + 0, // tp_weaklistoffset + nullptr, // tp_iter + nullptr, // tp_iternext + file_descriptor::Methods, // tp_methods + nullptr, // tp_members + file_descriptor::Getters, // tp_getset + &descriptor::PyBaseDescriptor_Type, // tp_base + nullptr, // tp_dict + nullptr, // tp_descr_get + nullptr, // tp_descr_set + 0, // tp_dictoffset + nullptr, // tp_init + nullptr, // tp_alloc + nullptr, // tp_new + PyObject_GC_Del, // tp_free }; PyObject* PyFileDescriptor_FromDescriptor( @@ -1634,11 +1659,15 @@ static PyMethodDef Methods[] = { PyTypeObject PyOneofDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME - ".OneofDescriptor", // tp_name - sizeof(PyBaseDescriptor), // tp_basicsize - 0, // tp_itemsize - nullptr, // tp_dealloc - 0, // tp_print + ".OneofDescriptor", // tp_name + sizeof(PyBaseDescriptor), // tp_basicsize + 0, // tp_itemsize + nullptr, // tp_dealloc +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif nullptr, // tp_getattr nullptr, // tp_setattr nullptr, // tp_compare @@ -1752,11 +1781,15 @@ static PyMethodDef Methods[] = { PyTypeObject PyServiceDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME - ".ServiceDescriptor", // tp_name - sizeof(PyBaseDescriptor), // tp_basicsize - 0, // tp_itemsize - nullptr, // tp_dealloc - 0, // tp_print + ".ServiceDescriptor", // tp_name + sizeof(PyBaseDescriptor), // tp_basicsize + 0, // tp_itemsize + nullptr, // tp_dealloc +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif nullptr, // tp_getattr nullptr, // tp_setattr nullptr, // tp_compare @@ -1876,11 +1909,15 @@ static PyMethodDef Methods[] = { PyTypeObject PyMethodDescriptor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME - ".MethodDescriptor", // tp_name - sizeof(PyBaseDescriptor), // tp_basicsize - 0, // tp_itemsize - nullptr, // tp_dealloc - 0, // tp_print + ".MethodDescriptor", // tp_name + sizeof(PyBaseDescriptor), // tp_basicsize + 0, // tp_itemsize + nullptr, // tp_dealloc +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif nullptr, // tp_getattr nullptr, // tp_setattr nullptr, // tp_compare diff --git a/python/google/protobuf/pyext/descriptor.h b/python/google/protobuf/pyext/descriptor.h index b99209dd19..d97e2f8103 100644 --- a/python/google/protobuf/pyext/descriptor.h +++ b/python/google/protobuf/pyext/descriptor.h @@ -33,6 +33,7 @@ #ifndef GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_H__ #define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_H__ +#define PY_SSIZE_T_CLEAN #include #include diff --git a/python/google/protobuf/pyext/descriptor_containers.cc b/python/google/protobuf/pyext/descriptor_containers.cc index adb1690947..a87155bc4e 100644 --- a/python/google/protobuf/pyext/descriptor_containers.cc +++ b/python/google/protobuf/pyext/descriptor_containers.cc @@ -49,6 +49,7 @@ // because the Python API is based on C, and does not play well with C++ // inheritance. +#define PY_SSIZE_T_CLEAN #include #include @@ -549,12 +550,16 @@ PyTypeObject DescriptorMapping_Type = { sizeof(PyContainer), // tp_basicsize 0, // tp_itemsize nullptr, // tp_dealloc - 0, // tp_pkrint - nullptr, // tp_getattr - nullptr, // tp_setattr - nullptr, // tp_compare - (reprfunc)ContainerRepr, // tp_repr - nullptr, // tp_as_number +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif + nullptr, // tp_getattr + nullptr, // tp_setattr + nullptr, // tp_compare + (reprfunc)ContainerRepr, // tp_repr + nullptr, // tp_as_number &MappingSequenceMethods, // tp_as_sequence &MappingMappingMethods, // tp_as_mapping nullptr, // tp_hash @@ -729,10 +734,14 @@ static PyMappingMethods SeqMappingMethods = { PyTypeObject DescriptorSequence_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "DescriptorSequence", // tp_name - sizeof(PyContainer), // tp_basicsize - 0, // tp_itemsize - nullptr, // tp_dealloc - 0, // tp_print + sizeof(PyContainer), // tp_basicsize + 0, // tp_itemsize + nullptr, // tp_dealloc +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif nullptr, // tp_getattr nullptr, // tp_setattr nullptr, // tp_compare @@ -874,25 +883,29 @@ static PyTypeObject ContainerIterator_Type = { sizeof(PyContainerIterator), // tp_basicsize 0, // tp_itemsize (destructor)Iterator_Dealloc, // tp_dealloc - 0, // tp_print - nullptr, // tp_getattr - nullptr, // tp_setattr - nullptr, // tp_compare - nullptr, // tp_repr - nullptr, // tp_as_number - nullptr, // tp_as_sequence - nullptr, // tp_as_mapping - nullptr, // tp_hash - nullptr, // tp_call - nullptr, // tp_str - nullptr, // tp_getattro - nullptr, // tp_setattro - nullptr, // tp_as_buffer - Py_TPFLAGS_DEFAULT, // tp_flags - nullptr, // tp_doc - nullptr, // tp_traverse - nullptr, // tp_clear - nullptr, // tp_richcompare +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif + nullptr, // tp_getattr + nullptr, // tp_setattr + nullptr, // tp_compare + nullptr, // tp_repr + nullptr, // tp_as_number + nullptr, // tp_as_sequence + nullptr, // tp_as_mapping + nullptr, // tp_hash + nullptr, // tp_call + nullptr, // tp_str + nullptr, // tp_getattro + nullptr, // tp_setattro + nullptr, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + nullptr, // tp_doc + nullptr, // tp_traverse + nullptr, // tp_clear + nullptr, // tp_richcompare 0, // tp_weaklistoffset PyObject_SelfIter, // tp_iter (iternextfunc)Iterator_Next, // tp_iternext diff --git a/python/google/protobuf/pyext/descriptor_containers.h b/python/google/protobuf/pyext/descriptor_containers.h index 4e05c58e2b..cf2cf4ad0b 100644 --- a/python/google/protobuf/pyext/descriptor_containers.h +++ b/python/google/protobuf/pyext/descriptor_containers.h @@ -34,6 +34,7 @@ // Mappings and Sequences of descriptors. // They implement containers like fields_by_name, EnumDescriptor.values... // See descriptor_containers.cc for more description. +#define PY_SSIZE_T_CLEAN #include namespace google { diff --git a/python/google/protobuf/pyext/descriptor_database.h b/python/google/protobuf/pyext/descriptor_database.h index d1bd7a3c19..5621a229ef 100644 --- a/python/google/protobuf/pyext/descriptor_database.h +++ b/python/google/protobuf/pyext/descriptor_database.h @@ -31,6 +31,7 @@ #ifndef GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_DATABASE_H__ #define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_DATABASE_H__ +#define PY_SSIZE_T_CLEAN #include #include diff --git a/python/google/protobuf/pyext/descriptor_pool.cc b/python/google/protobuf/pyext/descriptor_pool.cc index 5b0ceebfa4..2f39281830 100644 --- a/python/google/protobuf/pyext/descriptor_pool.cc +++ b/python/google/protobuf/pyext/descriptor_pool.cc @@ -32,6 +32,7 @@ #include +#define PY_SSIZE_T_CLEAN #include #include @@ -689,11 +690,15 @@ static PyMethodDef Methods[] = { PyTypeObject PyDescriptorPool_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME - ".DescriptorPool", // tp_name - sizeof(PyDescriptorPool), // tp_basicsize - 0, // tp_itemsize - cdescriptor_pool::Dealloc, // tp_dealloc - 0, // tp_print + ".DescriptorPool", // tp_name + sizeof(PyDescriptorPool), // tp_basicsize + 0, // tp_itemsize + cdescriptor_pool::Dealloc, // tp_dealloc +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif nullptr, // tp_getattr nullptr, // tp_setattr nullptr, // tp_compare diff --git a/python/google/protobuf/pyext/descriptor_pool.h b/python/google/protobuf/pyext/descriptor_pool.h index 48658d3e88..5d3c3a95cc 100644 --- a/python/google/protobuf/pyext/descriptor_pool.h +++ b/python/google/protobuf/pyext/descriptor_pool.h @@ -31,6 +31,7 @@ #ifndef GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_POOL_H__ #define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_POOL_H__ +#define PY_SSIZE_T_CLEAN #include #include diff --git a/python/google/protobuf/pyext/extension_dict.cc b/python/google/protobuf/pyext/extension_dict.cc index 760fc91f49..692029f682 100644 --- a/python/google/protobuf/pyext/extension_dict.cc +++ b/python/google/protobuf/pyext/extension_dict.cc @@ -367,12 +367,16 @@ static PyMethodDef Methods[] = { } // namespace extension_dict PyTypeObject ExtensionDict_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) // - FULL_MODULE_NAME ".ExtensionDict", // tp_name - sizeof(ExtensionDict), // tp_basicsize - 0, // tp_itemsize - (destructor)extension_dict::dealloc, // tp_dealloc - 0, // tp_print + PyVarObject_HEAD_INIT(&PyType_Type, 0) // + FULL_MODULE_NAME ".ExtensionDict", // tp_name + sizeof(ExtensionDict), // tp_basicsize + 0, // tp_itemsize + (destructor)extension_dict::dealloc, // tp_dealloc +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif nullptr, // tp_getattr nullptr, // tp_setattr nullptr, // tp_compare @@ -439,37 +443,41 @@ PyTypeObject ExtensionIterator_Type = { sizeof(extension_dict::ExtensionIterator), // tp_basicsize 0, // tp_itemsize extension_dict::DeallocExtensionIterator, // tp_dealloc - 0, // tp_print - nullptr, // tp_getattr - nullptr, // tp_setattr - nullptr, // tp_compare - nullptr, // tp_repr - nullptr, // tp_as_number - nullptr, // tp_as_sequence - nullptr, // tp_as_mapping - nullptr, // tp_hash - nullptr, // tp_call - nullptr, // tp_str - nullptr, // tp_getattro - nullptr, // tp_setattro - nullptr, // tp_as_buffer - Py_TPFLAGS_DEFAULT, // tp_flags - "A scalar map iterator", // tp_doc - nullptr, // tp_traverse - nullptr, // tp_clear - nullptr, // tp_richcompare - 0, // tp_weaklistoffset - PyObject_SelfIter, // tp_iter - IterNext, // tp_iternext - nullptr, // tp_methods - nullptr, // tp_members - nullptr, // tp_getset - nullptr, // tp_base - nullptr, // tp_dict - nullptr, // tp_descr_get - nullptr, // tp_descr_set - 0, // tp_dictoffset - nullptr, // tp_init +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif + nullptr, // tp_getattr + nullptr, // tp_setattr + nullptr, // tp_compare + nullptr, // tp_repr + nullptr, // tp_as_number + nullptr, // tp_as_sequence + nullptr, // tp_as_mapping + nullptr, // tp_hash + nullptr, // tp_call + nullptr, // tp_str + nullptr, // tp_getattro + nullptr, // tp_setattro + nullptr, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "A scalar map iterator", // tp_doc + nullptr, // tp_traverse + nullptr, // tp_clear + nullptr, // tp_richcompare + 0, // tp_weaklistoffset + PyObject_SelfIter, // tp_iter + IterNext, // tp_iternext + nullptr, // tp_methods + nullptr, // tp_members + nullptr, // tp_getset + nullptr, // tp_base + nullptr, // tp_dict + nullptr, // tp_descr_get + nullptr, // tp_descr_set + 0, // tp_dictoffset + nullptr, // tp_init }; } // namespace python } // namespace protobuf diff --git a/python/google/protobuf/pyext/extension_dict.h b/python/google/protobuf/pyext/extension_dict.h index c9da443161..a0581941bd 100644 --- a/python/google/protobuf/pyext/extension_dict.h +++ b/python/google/protobuf/pyext/extension_dict.h @@ -34,6 +34,7 @@ #ifndef GOOGLE_PROTOBUF_PYTHON_CPP_EXTENSION_DICT_H__ #define GOOGLE_PROTOBUF_PYTHON_CPP_EXTENSION_DICT_H__ +#define PY_SSIZE_T_CLEAN #include #include diff --git a/python/google/protobuf/pyext/field.cc b/python/google/protobuf/pyext/field.cc index 611ac8ac1e..0d3b0b9607 100644 --- a/python/google/protobuf/pyext/field.cc +++ b/python/google/protobuf/pyext/field.cc @@ -87,39 +87,43 @@ static PyTypeObject _CFieldProperty_Type = { sizeof(PyMessageFieldProperty), // tp_basicsize 0, // tp_itemsize nullptr, // tp_dealloc - 0, // tp_print - nullptr, // tp_getattr - nullptr, // tp_setattr - nullptr, // tp_compare - (reprfunc)field::Repr, // tp_repr - nullptr, // tp_as_number - nullptr, // tp_as_sequence - nullptr, // tp_as_mapping - nullptr, // tp_hash - nullptr, // tp_call - nullptr, // tp_str - nullptr, // tp_getattro - nullptr, // tp_setattro - nullptr, // tp_as_buffer - Py_TPFLAGS_DEFAULT, // tp_flags - "Field property of a Message", // tp_doc - nullptr, // tp_traverse - nullptr, // tp_clear - nullptr, // tp_richcompare - 0, // tp_weaklistoffset - nullptr, // tp_iter - nullptr, // tp_iternext - nullptr, // tp_methods - nullptr, // tp_members - field::Getters, // tp_getset - nullptr, // tp_base - nullptr, // tp_dict - (descrgetfunc)field::DescrGet, // tp_descr_get - (descrsetfunc)field::DescrSet, // tp_descr_set - 0, // tp_dictoffset - nullptr, // tp_init - nullptr, // tp_alloc - nullptr, // tp_new +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif + nullptr, // tp_getattr + nullptr, // tp_setattr + nullptr, // tp_compare + (reprfunc)field::Repr, // tp_repr + nullptr, // tp_as_number + nullptr, // tp_as_sequence + nullptr, // tp_as_mapping + nullptr, // tp_hash + nullptr, // tp_call + nullptr, // tp_str + nullptr, // tp_getattro + nullptr, // tp_setattro + nullptr, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "Field property of a Message", // tp_doc + nullptr, // tp_traverse + nullptr, // tp_clear + nullptr, // tp_richcompare + 0, // tp_weaklistoffset + nullptr, // tp_iter + nullptr, // tp_iternext + nullptr, // tp_methods + nullptr, // tp_members + field::Getters, // tp_getset + nullptr, // tp_base + nullptr, // tp_dict + (descrgetfunc)field::DescrGet, // tp_descr_get + (descrsetfunc)field::DescrSet, // tp_descr_set + 0, // tp_dictoffset + nullptr, // tp_init + nullptr, // tp_alloc + nullptr, // tp_new }; PyTypeObject* CFieldProperty_Type = &_CFieldProperty_Type; diff --git a/python/google/protobuf/pyext/field.h b/python/google/protobuf/pyext/field.h index 7b4660cab5..f9f94c4983 100644 --- a/python/google/protobuf/pyext/field.h +++ b/python/google/protobuf/pyext/field.h @@ -31,6 +31,7 @@ #ifndef GOOGLE_PROTOBUF_PYTHON_CPP_FIELD_H__ #define GOOGLE_PROTOBUF_PYTHON_CPP_FIELD_H__ +#define PY_SSIZE_T_CLEAN #include namespace google { diff --git a/python/google/protobuf/pyext/map_container.cc b/python/google/protobuf/pyext/map_container.cc index c953c6594f..4d516d2917 100644 --- a/python/google/protobuf/pyext/map_container.cc +++ b/python/google/protobuf/pyext/map_container.cc @@ -432,12 +432,12 @@ int MapReflectionFriend::ScalarMapSetItem(PyObject* _self, PyObject* key, return -1; } - self->version++; - if (v) { // Set item to v. - reflection->InsertOrLookupMapValue(message, self->parent_field_descriptor, - map_key, &value); + if (reflection->InsertOrLookupMapValue( + message, self->parent_field_descriptor, map_key, &value)) { + self->version++; + } if (!PythonToMapValueRef(self, v, reflection->SupportsUnknownEnumValues(), &value)) { @@ -448,6 +448,7 @@ int MapReflectionFriend::ScalarMapSetItem(PyObject* _self, PyObject* key, // Delete key from map. if (reflection->DeleteMapValue(message, self->parent_field_descriptor, map_key)) { + self->version++; return 0; } else { PyErr_Format(PyExc_KeyError, "Key not present in map"); @@ -853,11 +854,15 @@ static void DeallocMapIterator(PyObject* _self) { PyTypeObject MapIterator_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME - ".MapIterator", // tp_name - sizeof(MapIterator), // tp_basicsize - 0, // tp_itemsize - DeallocMapIterator, // tp_dealloc - 0, // tp_print + ".MapIterator", // tp_name + sizeof(MapIterator), // tp_basicsize + 0, // tp_itemsize + DeallocMapIterator, // tp_dealloc +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif nullptr, // tp_getattr nullptr, // tp_setattr nullptr, // tp_compare diff --git a/python/google/protobuf/pyext/map_container.h b/python/google/protobuf/pyext/map_container.h index 842602e79f..e14136efab 100644 --- a/python/google/protobuf/pyext/map_container.h +++ b/python/google/protobuf/pyext/map_container.h @@ -31,6 +31,7 @@ #ifndef GOOGLE_PROTOBUF_PYTHON_CPP_MAP_CONTAINER_H__ #define GOOGLE_PROTOBUF_PYTHON_CPP_MAP_CONTAINER_H__ +#define PY_SSIZE_T_CLEAN #include #include diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index 3a3382494a..35594ec99c 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc @@ -454,11 +454,15 @@ static bool allow_oversize_protos = false; static PyTypeObject _CMessageClass_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME - ".MessageMeta", // tp_name - sizeof(CMessageClass), // tp_basicsize - 0, // tp_itemsize - message_meta::Dealloc, // tp_dealloc - 0, // tp_print + ".MessageMeta", // tp_name + sizeof(CMessageClass), // tp_basicsize + 0, // tp_itemsize + message_meta::Dealloc, // tp_dealloc +#if PY_VERSION_HEX < 0x03080000 + nullptr, /* tp_print */ +#else + 0, /* tp_vectorcall_offset */ +#endif nullptr, // tp_getattr nullptr, // tp_setattr nullptr, // tp_compare @@ -1930,7 +1934,7 @@ static PyObject* MergeFromString(CMessage* self, PyObject* arg) { "Error parsing message as the message exceeded the protobuf limit " "with type '%s'", self->GetMessageClass()->message_descriptor->full_name().c_str()); - return NULL; + return nullptr; } // ctx has an explicit limit set (length of string_view), so we have to @@ -2735,20 +2739,24 @@ static CMessageClass _CMessage_Type = {{{ sizeof(CMessage), // tp_basicsize 0, // tp_itemsize (destructor)cmessage::Dealloc, // tp_dealloc - 0, // tp_print - nullptr, // tp_getattr - nullptr, // tp_setattr - nullptr, // tp_compare - (reprfunc)cmessage::ToStr, // tp_repr - nullptr, // tp_as_number - nullptr, // tp_as_sequence - nullptr, // tp_as_mapping - PyObject_HashNotImplemented, // tp_hash - nullptr, // tp_call - (reprfunc)cmessage::ToStr, // tp_str - cmessage::GetAttr, // tp_getattro - nullptr, // tp_setattro - nullptr, // tp_as_buffer +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif + nullptr, // tp_getattr + nullptr, // tp_setattr + nullptr, // tp_compare + (reprfunc)cmessage::ToStr, // tp_repr + nullptr, // tp_as_number + nullptr, // tp_as_sequence + nullptr, // tp_as_mapping + PyObject_HashNotImplemented, // tp_hash + nullptr, // tp_call + (reprfunc)cmessage::ToStr, // tp_str + cmessage::GetAttr, // tp_getattro + nullptr, // tp_setattro + nullptr, // tp_as_buffer Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_VERSION_TAG, // tp_flags "A ProtocolMessage", // tp_doc diff --git a/python/google/protobuf/pyext/message.h b/python/google/protobuf/pyext/message.h index 29af255154..b17daa5806 100644 --- a/python/google/protobuf/pyext/message.h +++ b/python/google/protobuf/pyext/message.h @@ -34,6 +34,7 @@ #ifndef GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_H__ #define GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_H__ +#define PY_SSIZE_T_CLEAN #include #include diff --git a/python/google/protobuf/pyext/message_factory.cc b/python/google/protobuf/pyext/message_factory.cc index 64f9fed5e5..bc44dd4572 100644 --- a/python/google/protobuf/pyext/message_factory.cc +++ b/python/google/protobuf/pyext/message_factory.cc @@ -30,6 +30,7 @@ #include +#define PY_SSIZE_T_CLEAN #include #include @@ -252,20 +253,24 @@ PyTypeObject PyMessageFactory_Type = { sizeof(PyMessageFactory), // tp_basicsize 0, // tp_itemsize message_factory::Dealloc, // tp_dealloc - 0, // tp_print - nullptr, // tp_getattr - nullptr, // tp_setattr - nullptr, // tp_compare - nullptr, // tp_repr - nullptr, // tp_as_number - nullptr, // tp_as_sequence - nullptr, // tp_as_mapping - nullptr, // tp_hash - nullptr, // tp_call - nullptr, // tp_str - nullptr, // tp_getattro - nullptr, // tp_setattro - nullptr, // tp_as_buffer +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif + nullptr, // tp_getattr + nullptr, // tp_setattr + nullptr, // tp_compare + nullptr, // tp_repr + nullptr, // tp_as_number + nullptr, // tp_as_sequence + nullptr, // tp_as_mapping + nullptr, // tp_hash + nullptr, // tp_call + nullptr, // tp_str + nullptr, // tp_getattro + nullptr, // tp_setattro + nullptr, // tp_as_buffer Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, // tp_flags "A static Message Factory", // tp_doc message_factory::GcTraverse, // tp_traverse diff --git a/python/google/protobuf/pyext/message_factory.h b/python/google/protobuf/pyext/message_factory.h index 515c29cdb8..7dfe425dd5 100644 --- a/python/google/protobuf/pyext/message_factory.h +++ b/python/google/protobuf/pyext/message_factory.h @@ -31,6 +31,7 @@ #ifndef GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_FACTORY_H__ #define GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_FACTORY_H__ +#define PY_SSIZE_T_CLEAN #include #include diff --git a/python/google/protobuf/pyext/message_module.cc b/python/google/protobuf/pyext/message_module.cc index 1f953fc783..2d3c1d2087 100644 --- a/python/google/protobuf/pyext/message_module.cc +++ b/python/google/protobuf/pyext/message_module.cc @@ -28,6 +28,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#define PY_SSIZE_T_CLEAN #include #include diff --git a/python/google/protobuf/pyext/repeated_composite_container.h b/python/google/protobuf/pyext/repeated_composite_container.h index e241827ef5..30536eabd4 100644 --- a/python/google/protobuf/pyext/repeated_composite_container.h +++ b/python/google/protobuf/pyext/repeated_composite_container.h @@ -34,6 +34,7 @@ #ifndef GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_COMPOSITE_CONTAINER_H__ #define GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_COMPOSITE_CONTAINER_H__ +#define PY_SSIZE_T_CLEAN #include #include diff --git a/python/google/protobuf/pyext/repeated_scalar_container.h b/python/google/protobuf/pyext/repeated_scalar_container.h index f9f0ea8f31..372b6f0a35 100644 --- a/python/google/protobuf/pyext/repeated_scalar_container.h +++ b/python/google/protobuf/pyext/repeated_scalar_container.h @@ -34,6 +34,7 @@ #ifndef GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_SCALAR_CONTAINER_H__ #define GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_SCALAR_CONTAINER_H__ +#define PY_SSIZE_T_CLEAN #include #include diff --git a/python/google/protobuf/pyext/scoped_pyobject_ptr.h b/python/google/protobuf/pyext/scoped_pyobject_ptr.h index e6ea7e33e5..f684152749 100644 --- a/python/google/protobuf/pyext/scoped_pyobject_ptr.h +++ b/python/google/protobuf/pyext/scoped_pyobject_ptr.h @@ -35,6 +35,7 @@ #include +#define PY_SSIZE_T_CLEAN #include namespace google { namespace protobuf { diff --git a/python/google/protobuf/pyext/unknown_fields.cc b/python/google/protobuf/pyext/unknown_fields.cc index fbfee3d422..dcd63b2e29 100644 --- a/python/google/protobuf/pyext/unknown_fields.cc +++ b/python/google/protobuf/pyext/unknown_fields.cc @@ -30,6 +30,7 @@ #include +#define PY_SSIZE_T_CLEAN #include #include #include @@ -155,11 +156,15 @@ static PySequenceMethods SqMethods = { PyTypeObject PyUnknownFields_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME - ".PyUnknownFields", // tp_name - sizeof(PyUnknownFields), // tp_basicsize - 0, // tp_itemsize - unknown_fields::Dealloc, // tp_dealloc - 0, // tp_print + ".PyUnknownFields", // tp_name + sizeof(PyUnknownFields), // tp_basicsize + 0, // tp_itemsize + unknown_fields::Dealloc, // tp_dealloc +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif nullptr, // tp_getattr nullptr, // tp_setattr nullptr, // tp_compare @@ -219,7 +224,7 @@ const UnknownField* GetUnknownField(PyUnknownFieldRef* self) { "The parent message might be cleared."); return nullptr; } - ssize_t total_size = fields->field_count(); + Py_ssize_t total_size = fields->field_count(); if (self->index >= total_size) { PyErr_Format(PyExc_ValueError, "UnknownField does not exist. " @@ -312,11 +317,15 @@ static PyGetSetDef Getters[] = { PyTypeObject PyUnknownFieldRef_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME - ".PyUnknownFieldRef", // tp_name - sizeof(PyUnknownFieldRef), // tp_basicsize - 0, // tp_itemsize - unknown_field::Dealloc, // tp_dealloc - 0, // tp_print + ".PyUnknownFieldRef", // tp_name + sizeof(PyUnknownFieldRef), // tp_basicsize + 0, // tp_itemsize + unknown_field::Dealloc, // tp_dealloc +#if PY_VERSION_HEX < 0x03080000 + nullptr, // tp_print +#else + 0, // tp_vectorcall_offset +#endif nullptr, // tp_getattr nullptr, // tp_setattr nullptr, // tp_compare diff --git a/python/google/protobuf/pyext/unknown_fields.h b/python/google/protobuf/pyext/unknown_fields.h index 94d55e148d..e7b0b35c41 100644 --- a/python/google/protobuf/pyext/unknown_fields.h +++ b/python/google/protobuf/pyext/unknown_fields.h @@ -31,6 +31,7 @@ #ifndef GOOGLE_PROTOBUF_PYTHON_CPP_UNKNOWN_FIELDS_H__ #define GOOGLE_PROTOBUF_PYTHON_CPP_UNKNOWN_FIELDS_H__ +#define PY_SSIZE_T_CLEAN #include #include diff --git a/python/google/protobuf/python_protobuf.h b/python/google/protobuf/python_protobuf.h index 8db1ffb750..4fcf065404 100644 --- a/python/google/protobuf/python_protobuf.h +++ b/python/google/protobuf/python_protobuf.h @@ -36,6 +36,7 @@ #ifndef GOOGLE_PROTOBUF_PYTHON_PYTHON_PROTOBUF_H__ #define GOOGLE_PROTOBUF_PYTHON_PYTHON_PROTOBUF_H__ +#define PY_SSIZE_T_CLEAN #include namespace google { diff --git a/python/google/protobuf/service_reflection.py b/python/google/protobuf/service_reflection.py index 75c51ff322..f82ab7145a 100644 --- a/python/google/protobuf/service_reflection.py +++ b/python/google/protobuf/service_reflection.py @@ -133,7 +133,7 @@ class _ServiceBuilder(object): """ self.descriptor = service_descriptor - def BuildService(self, cls): + def BuildService(builder, cls): """Constructs the service class. Args: @@ -143,18 +143,26 @@ class _ServiceBuilder(object): # CallMethod needs to operate with an instance of the Service class. This # internal wrapper function exists only to be able to pass the service # instance to the method that does the real CallMethod work. - def _WrapCallMethod(srvc, method_descriptor, - rpc_controller, request, callback): - return self._CallMethod(srvc, method_descriptor, - rpc_controller, request, callback) - self.cls = cls + # Making sure to use exact argument names from the abstract interface in + # service.py to match the type signature + def _WrapCallMethod(self, method_descriptor, rpc_controller, request, done): + return builder._CallMethod(self, method_descriptor, rpc_controller, + request, done) + + def _WrapGetRequestClass(self, method_descriptor): + return builder._GetRequestClass(method_descriptor) + + def _WrapGetResponseClass(self, method_descriptor): + return builder._GetResponseClass(method_descriptor) + + builder.cls = cls cls.CallMethod = _WrapCallMethod - cls.GetDescriptor = staticmethod(lambda: self.descriptor) - cls.GetDescriptor.__doc__ = "Returns the service descriptor." - cls.GetRequestClass = self._GetRequestClass - cls.GetResponseClass = self._GetResponseClass - for method in self.descriptor.methods: - setattr(cls, method.name, self._GenerateNonImplementedMethod(method)) + cls.GetDescriptor = staticmethod(lambda: builder.descriptor) + cls.GetDescriptor.__doc__ = 'Returns the service descriptor.' + cls.GetRequestClass = _WrapGetRequestClass + cls.GetResponseClass = _WrapGetResponseClass + for method in builder.descriptor.methods: + setattr(cls, method.name, builder._GenerateNonImplementedMethod(method)) def _CallMethod(self, srvc, method_descriptor, rpc_controller, request, callback): diff --git a/python/setup.py b/python/setup.py index ab42b6fe17..fb35cf5176 100755 --- a/python/setup.py +++ b/python/setup.py @@ -1,6 +1,9 @@ #! /usr/bin/env python # # See README for usage instructions. + +# pylint:disable=missing-module-docstring +# pylint:disable=g-bad-import-order from distutils import util import fnmatch import glob @@ -10,7 +13,9 @@ import re import subprocess import sys import sysconfig -import platform + +# pylint:disable=g-importing-member +# pylint:disable=g-multiple-import # We must use setuptools, not distutils, because we need to use the # namespace_packages option for the "google" package. @@ -24,44 +29,52 @@ from distutils.spawn import find_executable # Find the Protocol Compiler. if 'PROTOC' in os.environ and os.path.exists(os.environ['PROTOC']): protoc = os.environ['PROTOC'] -elif os.path.exists("../src/protoc"): - protoc = "../src/protoc" -elif os.path.exists("../src/protoc.exe"): - protoc = "../src/protoc.exe" -elif os.path.exists("../vsprojects/Debug/protoc.exe"): - protoc = "../vsprojects/Debug/protoc.exe" -elif os.path.exists("../vsprojects/Release/protoc.exe"): - protoc = "../vsprojects/Release/protoc.exe" +elif os.path.exists('../src/protoc'): + protoc = '../src/protoc' +elif os.path.exists('../src/protoc.exe'): + protoc = '../src/protoc.exe' +elif os.path.exists('../vsprojects/Debug/protoc.exe'): + protoc = '../vsprojects/Debug/protoc.exe' +elif os.path.exists('../vsprojects/Release/protoc.exe'): + protoc = '../vsprojects/Release/protoc.exe' else: - protoc = find_executable("protoc") + protoc = find_executable('protoc') def GetVersion(): - """Gets the version from google/protobuf/__init__.py + """Reads and returns the version from google/protobuf/__init__.py. Do not import google.protobuf.__init__ directly, because an installed - protobuf library may be loaded instead.""" + protobuf library may be loaded instead. + + Returns: + The version. + """ with open(os.path.join('google', 'protobuf', '__init__.py')) as version_file: - exec(version_file.read(), globals()) - global __version__ - return __version__ + exec(version_file.read(), globals()) # pylint:disable=exec-used + return __version__ # pylint:disable=undefined-variable + +def GenProto(source, require=True): + """Generates a _pb2.py from the given .proto file. -def generate_proto(source, require = True): - """Invokes the Protocol Compiler to generate a _pb2.py from the given - .proto file. Does nothing if the output already exists and is newer than - the input.""" + Does nothing if the output already exists and is newer than the input. + + Args: + source: the .proto file path. + require: if True, exit immediately when a path is not found. + """ if not require and not os.path.exists(source): return - output = source.replace(".proto", "_pb2.py").replace("../src/", "") + output = source.replace('.proto', '_pb2.py').replace('../src/', '') if (not os.path.exists(output) or (os.path.exists(source) and os.path.getmtime(source) > os.path.getmtime(output))): - print("Generating %s..." % output) + print('Generating %s...' % output) if not os.path.exists(source): sys.stderr.write("Can't find required file: %s\n" % source) @@ -69,78 +82,85 @@ def generate_proto(source, require = True): if protoc is None: sys.stderr.write( - "protoc is not installed nor found in ../src. Please compile it " - "or install the binary package.\n") + 'protoc is not installed nor found in ../src. Please compile it ' + 'or install the binary package.\n') sys.exit(-1) - protoc_command = [ protoc, "-I../src", "-I.", "--python_out=.", source ] + protoc_command = [protoc, '-I../src', '-I.', '--python_out=.', source] if subprocess.call(protoc_command) != 0: sys.exit(-1) + def GenerateUnittestProtos(): - generate_proto("../src/google/protobuf/any_test.proto", False) - generate_proto("../src/google/protobuf/map_proto2_unittest.proto", False) - generate_proto("../src/google/protobuf/map_unittest.proto", False) - generate_proto("../src/google/protobuf/test_messages_proto3.proto", False) - generate_proto("../src/google/protobuf/test_messages_proto2.proto", False) - generate_proto("../src/google/protobuf/unittest_arena.proto", False) - generate_proto("../src/google/protobuf/unittest.proto", False) - generate_proto("../src/google/protobuf/unittest_custom_options.proto", False) - generate_proto("../src/google/protobuf/unittest_import.proto", False) - generate_proto("../src/google/protobuf/unittest_import_public.proto", False) - generate_proto("../src/google/protobuf/unittest_mset.proto", False) - generate_proto("../src/google/protobuf/unittest_mset_wire_format.proto", False) - generate_proto("../src/google/protobuf/unittest_no_generic_services.proto", False) - generate_proto("../src/google/protobuf/unittest_proto3_arena.proto", False) - generate_proto("../src/google/protobuf/util/json_format.proto", False) - generate_proto("../src/google/protobuf/util/json_format_proto3.proto", False) - generate_proto("google/protobuf/internal/any_test.proto", False) - generate_proto("google/protobuf/internal/descriptor_pool_test1.proto", False) - generate_proto("google/protobuf/internal/descriptor_pool_test2.proto", False) - generate_proto("google/protobuf/internal/factory_test1.proto", False) - generate_proto("google/protobuf/internal/factory_test2.proto", False) - generate_proto("google/protobuf/internal/file_options_test.proto", False) - generate_proto("google/protobuf/internal/import_test_package/inner.proto", False) - generate_proto("google/protobuf/internal/import_test_package/outer.proto", False) - generate_proto("google/protobuf/internal/missing_enum_values.proto", False) - generate_proto("google/protobuf/internal/message_set_extensions.proto", False) - generate_proto("google/protobuf/internal/more_extensions.proto", False) - generate_proto("google/protobuf/internal/more_extensions_dynamic.proto", False) - generate_proto("google/protobuf/internal/more_messages.proto", False) - generate_proto("google/protobuf/internal/no_package.proto", False) - generate_proto("google/protobuf/internal/packed_field_test.proto", False) - generate_proto("google/protobuf/internal/test_bad_identifiers.proto", False) - generate_proto("google/protobuf/internal/test_proto3_optional.proto", False) - generate_proto("google/protobuf/pyext/python.proto", False) - - -class clean(_clean): + """Generates protobuf code for unittests.""" + GenProto('../src/google/protobuf/any_test.proto', False) + GenProto('../src/google/protobuf/map_proto2_unittest.proto', False) + GenProto('../src/google/protobuf/map_unittest.proto', False) + GenProto('../src/google/protobuf/test_messages_proto3.proto', False) + GenProto('../src/google/protobuf/test_messages_proto2.proto', False) + GenProto('../src/google/protobuf/unittest_arena.proto', False) + GenProto('../src/google/protobuf/unittest.proto', False) + GenProto('../src/google/protobuf/unittest_custom_options.proto', False) + GenProto('../src/google/protobuf/unittest_import.proto', False) + GenProto('../src/google/protobuf/unittest_import_public.proto', False) + GenProto('../src/google/protobuf/unittest_mset.proto', False) + GenProto('../src/google/protobuf/unittest_mset_wire_format.proto', False) + GenProto('../src/google/protobuf/unittest_no_generic_services.proto', False) + GenProto('../src/google/protobuf/unittest_proto3_arena.proto', False) + GenProto('../src/google/protobuf/util/json_format.proto', False) + GenProto('../src/google/protobuf/util/json_format_proto3.proto', False) + GenProto('google/protobuf/internal/any_test.proto', False) + GenProto('google/protobuf/internal/descriptor_pool_test1.proto', False) + GenProto('google/protobuf/internal/descriptor_pool_test2.proto', False) + GenProto('google/protobuf/internal/factory_test1.proto', False) + GenProto('google/protobuf/internal/factory_test2.proto', False) + GenProto('google/protobuf/internal/file_options_test.proto', False) + GenProto('google/protobuf/internal/import_test_package/inner.proto', False) + GenProto('google/protobuf/internal/import_test_package/outer.proto', False) + GenProto('google/protobuf/internal/missing_enum_values.proto', False) + GenProto('google/protobuf/internal/message_set_extensions.proto', False) + GenProto('google/protobuf/internal/more_extensions.proto', False) + GenProto('google/protobuf/internal/more_extensions_dynamic.proto', False) + GenProto('google/protobuf/internal/more_messages.proto', False) + GenProto('google/protobuf/internal/no_package.proto', False) + GenProto('google/protobuf/internal/packed_field_test.proto', False) + GenProto('google/protobuf/internal/test_bad_identifiers.proto', False) + GenProto('google/protobuf/internal/test_proto3_optional.proto', False) + GenProto('google/protobuf/pyext/python.proto', False) + + +class CleanCmd(_clean): + """Custom clean command for building the protobuf extension.""" + def run(self): # Delete generated files in the code tree. - for (dirpath, dirnames, filenames) in os.walk("."): + for (dirpath, unused_dirnames, filenames) in os.walk('.'): for filename in filenames: filepath = os.path.join(dirpath, filename) - if filepath.endswith("_pb2.py") or filepath.endswith(".pyc") or \ - filepath.endswith(".so") or filepath.endswith(".o"): + if (filepath.endswith('_pb2.py') or filepath.endswith('.pyc') or + filepath.endswith('.so') or filepath.endswith('.o')): os.remove(filepath) # _clean is an old-style class, so super() doesn't work. _clean.run(self) -class build_py(_build_py): + +class BuildPyCmd(_build_py): + """Custom build_py command for building the protobuf runtime.""" + def run(self): # Generate necessary .proto file if it doesn't exist. - generate_proto("../src/google/protobuf/descriptor.proto") - generate_proto("../src/google/protobuf/compiler/plugin.proto") - generate_proto("../src/google/protobuf/any.proto") - generate_proto("../src/google/protobuf/api.proto") - generate_proto("../src/google/protobuf/duration.proto") - generate_proto("../src/google/protobuf/empty.proto") - generate_proto("../src/google/protobuf/field_mask.proto") - generate_proto("../src/google/protobuf/source_context.proto") - generate_proto("../src/google/protobuf/struct.proto") - generate_proto("../src/google/protobuf/timestamp.proto") - generate_proto("../src/google/protobuf/type.proto") - generate_proto("../src/google/protobuf/wrappers.proto") + GenProto('../src/google/protobuf/descriptor.proto') + GenProto('../src/google/protobuf/compiler/plugin.proto') + GenProto('../src/google/protobuf/any.proto') + GenProto('../src/google/protobuf/api.proto') + GenProto('../src/google/protobuf/duration.proto') + GenProto('../src/google/protobuf/empty.proto') + GenProto('../src/google/protobuf/field_mask.proto') + GenProto('../src/google/protobuf/source_context.proto') + GenProto('../src/google/protobuf/struct.proto') + GenProto('../src/google/protobuf/timestamp.proto') + GenProto('../src/google/protobuf/type.proto') + GenProto('../src/google/protobuf/wrappers.proto') GenerateUnittestProtos() # _build_py is an old-style class, so super() doesn't work. @@ -148,17 +168,18 @@ class build_py(_build_py): def find_package_modules(self, package, package_dir): exclude = ( - "*test*", - "google/protobuf/internal/*_pb2.py", - "google/protobuf/internal/_parameterized.py", - "google/protobuf/pyext/python_pb2.py", + '*test*', + 'google/protobuf/internal/*_pb2.py', + 'google/protobuf/internal/_parameterized.py', + 'google/protobuf/pyext/python_pb2.py', ) modules = _build_py.find_package_modules(self, package, package_dir) return [(pkg, mod, fil) for (pkg, mod, fil) in modules if not any(fnmatch.fnmatchcase(fil, pat=pat) for pat in exclude)] -class build_ext(_build_ext): +class BuildExtCmd(_build_ext): + """Command class for building the protobuf Python extension.""" def get_ext_filename(self, ext_name): # since python3.5, python extensions' shared libraries use a suffix that @@ -169,23 +190,25 @@ class build_ext(_build_ext): # suffix so that the resulting file name matches the target architecture # and we end up with a well-formed wheel. filename = _build_ext.get_ext_filename(self, ext_name) - orig_ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") - new_ext_suffix = os.getenv("PROTOCOL_BUFFERS_OVERRIDE_EXT_SUFFIX") + orig_ext_suffix = sysconfig.get_config_var('EXT_SUFFIX') + new_ext_suffix = os.getenv('PROTOCOL_BUFFERS_OVERRIDE_EXT_SUFFIX') if new_ext_suffix and filename.endswith(orig_ext_suffix): filename = filename[:-len(orig_ext_suffix)] + new_ext_suffix return filename -class test_conformance(_build_py): + +class TestConformanceCmd(_build_py): target = 'test_python' + def run(self): # Python 2.6 dodges these extra failures. - os.environ["CONFORMANCE_PYTHON_EXTRA_FAILURES"] = ( - "--failure_list failure_list_python-post26.txt") - cmd = 'cd ../conformance && make %s' % (test_conformance.target) - status = subprocess.check_call(cmd, shell=True) + os.environ['CONFORMANCE_PYTHON_EXTRA_FAILURES'] = ( + '--failure_list failure_list_python-post26.txt') + cmd = 'cd ../conformance && make %s' % (TestConformanceCmd.target) + subprocess.check_call(cmd, shell=True) -def get_option_from_sys_argv(option_str): +def GetOptionFromArgv(option_str): if option_str in sys.argv: sys.argv.remove(option_str) return True @@ -195,21 +218,37 @@ def get_option_from_sys_argv(option_str): if __name__ == '__main__': ext_module_list = [] warnings_as_errors = '--warnings_as_errors' - if get_option_from_sys_argv('--cpp_implementation'): + if GetOptionFromArgv('--cpp_implementation'): # Link libprotobuf.a and libprotobuf-lite.a statically with the # extension. Note that those libraries have to be compiled with # -fPIC for this to work. - compile_static_ext = get_option_from_sys_argv('--compile_static_extension') + compile_static_ext = GetOptionFromArgv('--compile_static_extension') libraries = ['protobuf'] extra_objects = None if compile_static_ext: libraries = None extra_objects = ['../src/.libs/libprotobuf.a', '../src/.libs/libprotobuf-lite.a'] - test_conformance.target = 'test_python_cpp' + TestConformanceCmd.target = 'test_python_cpp' extra_compile_args = [] + message_extra_link_args = None + api_implementation_link_args = None + if 'darwin' in sys.platform: + if sys.version_info[0] == 2: + message_init_symbol = 'init_message' + api_implementation_init_symbol = 'init_api_implementation' + else: + message_init_symbol = 'PyInit__message' + api_implementation_init_symbol = 'PyInit__api_implementation' + message_extra_link_args = [ + '-Wl,-exported_symbol,_%s' % message_init_symbol + ] + api_implementation_link_args = [ + '-Wl,-exported_symbol,_%s' % api_implementation_init_symbol + ] + if sys.platform != 'win32': extra_compile_args.append('-Wno-write-strings') extra_compile_args.append('-Wno-invalid-offsetof') @@ -218,8 +257,8 @@ if __name__ == '__main__': extra_compile_args.append('-std=c++11') if sys.platform == 'darwin': - extra_compile_args.append("-Wno-shorten-64-to-32"); - extra_compile_args.append("-Wno-deprecated-register"); + extra_compile_args.append('-Wno-shorten-64-to-32') + extra_compile_args.append('-Wno-deprecated-register') # https://developer.apple.com/documentation/xcode_release_notes/xcode_10_release_notes # C++ projects must now migrate to libc++ and are recommended to set a @@ -227,7 +266,7 @@ if __name__ == '__main__': if sys.platform == 'darwin': mac_target = str(sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')) if mac_target and (pkg_resources.parse_version(mac_target) < - pkg_resources.parse_version('10.9.0')): + pkg_resources.parse_version('10.9.0')): os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.9' os.environ['_PYTHON_HOST_PLATFORM'] = re.sub( r'macosx-[0-9]+\.[0-9]+-(.+)', r'macosx-10.9-\1', @@ -242,10 +281,10 @@ if __name__ == '__main__': extra_compile_args.append('-DMS_WIN64') # MSVS default is dymanic - if (sys.platform == 'win32'): + if sys.platform == 'win32': extra_compile_args.append('/MT') - if "clang" in os.popen('$CC --version 2> /dev/null').read(): + if 'clang' in os.popen('$CC --version 2> /dev/null').read(): extra_compile_args.append('-Wno-shorten-64-to-32') if warnings_as_errors in sys.argv: @@ -255,18 +294,21 @@ if __name__ == '__main__': # C++ implementation extension ext_module_list.extend([ Extension( - "google.protobuf.pyext._message", + 'google.protobuf.pyext._message', glob.glob('google/protobuf/pyext/*.cc'), - include_dirs=[".", "../src"], + include_dirs=['.', '../src'], libraries=libraries, extra_objects=extra_objects, + extra_link_args=message_extra_link_args, library_dirs=['../src/.libs'], extra_compile_args=extra_compile_args, ), Extension( - "google.protobuf.internal._api_implementation", + 'google.protobuf.internal._api_implementation', glob.glob('google/protobuf/internal/api_implementation.cc'), - extra_compile_args=extra_compile_args + ['-DPYTHON_PROTO2_CPP_IMPL_V2'], + extra_compile_args=(extra_compile_args + + ['-DPYTHON_PROTO2_CPP_IMPL_V2']), + extra_link_args=api_implementation_link_args, ), ]) os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp' @@ -283,15 +325,14 @@ if __name__ == '__main__': url='https://developers.google.com/protocol-buffers/', maintainer='protobuf@googlegroups.com', maintainer_email='protobuf@googlegroups.com', - license='3-Clause BSD License', + license='BSD-3-Clause', classifiers=[ - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], namespace_packages=['google'], packages=find_packages( @@ -301,12 +342,12 @@ if __name__ == '__main__': ],), test_suite='google.protobuf.internal', cmdclass={ - 'clean': clean, - 'build_py': build_py, - 'build_ext': build_ext, - 'test_conformance': test_conformance, + 'clean': CleanCmd, + 'build_py': BuildPyCmd, + 'build_ext': BuildExtCmd, + 'test_conformance': TestConformanceCmd, }, install_requires=install_requires, ext_modules=ext_module_list, - python_requires=">=3.5", + python_requires='>=3.7', ) diff --git a/ruby/ext/google/protobuf_c/convert.c b/ruby/ext/google/protobuf_c/convert.c index 8b98aeeb56..2b3a6501b0 100644 --- a/ruby/ext/google/protobuf_c/convert.c +++ b/ruby/ext/google/protobuf_c/convert.c @@ -162,9 +162,9 @@ upb_MessageValue Convert_RubyToUpb(VALUE value, const char* name, } case kUpb_CType_String: { VALUE utf8 = rb_enc_from_encoding(rb_utf8_encoding()); - if (CLASS_OF(value) == rb_cSymbol) { + if (rb_obj_class(value) == rb_cSymbol) { value = rb_funcall(value, rb_intern("to_s"), 0); - } else if (CLASS_OF(value) != rb_cString) { + } else if (rb_obj_class(value) != rb_cString) { rb_raise(cTypeError, "Invalid argument for string field '%s' (given %s).", name, rb_class2name(CLASS_OF(value))); @@ -185,7 +185,7 @@ upb_MessageValue Convert_RubyToUpb(VALUE value, const char* name, } case kUpb_CType_Bytes: { VALUE bytes = rb_enc_from_encoding(rb_ascii8bit_encoding()); - if (CLASS_OF(value) != rb_cString) { + if (rb_obj_class(value) != rb_cString) { rb_raise(cTypeError, "Invalid argument for bytes field '%s' (given %s).", name, rb_class2name(CLASS_OF(value))); diff --git a/ruby/ext/google/protobuf_c/message.c b/ruby/ext/google/protobuf_c/message.c index 7feee75db8..b889230843 100644 --- a/ruby/ext/google/protobuf_c/message.c +++ b/ruby/ext/google/protobuf_c/message.c @@ -953,13 +953,35 @@ static VALUE Message_index_set(VALUE _self, VALUE field_name, VALUE value) { /* * call-seq: - * MessageClass.decode(data) => message + * MessageClass.decode(data, options) => message * * Decodes the given data (as a string containing bytes in protocol buffers wire * format) under the interpretration given by this message class's definition * and returns a message object with the corresponding field values. + * @param options [Hash] options for the decoder + * recursion_limit: set to maximum decoding depth for message (default is 64) */ -static VALUE Message_decode(VALUE klass, VALUE data) { +static VALUE Message_decode(int argc, VALUE* argv, VALUE klass) { + VALUE data = argv[0]; + int options = 0; + + if (argc < 1 || argc > 2) { + rb_raise(rb_eArgError, "Expected 1 or 2 arguments."); + } + + if (argc == 2) { + VALUE hash_args = argv[1]; + if (TYPE(hash_args) != T_HASH) { + rb_raise(rb_eArgError, "Expected hash arguments."); + } + + VALUE depth = rb_hash_lookup(hash_args, ID2SYM(rb_intern("recursion_limit"))); + + if (depth != Qnil && TYPE(depth) == T_FIXNUM) { + options |= UPB_DECODE_MAXDEPTH(FIX2INT(depth)); + } + } + if (TYPE(data) != T_STRING) { rb_raise(rb_eArgError, "Expected string for binary protobuf data."); } @@ -969,7 +991,7 @@ static VALUE Message_decode(VALUE klass, VALUE data) { upb_DecodeStatus status = upb_Decode( RSTRING_PTR(data), RSTRING_LEN(data), (upb_Message*)msg->msg, - upb_MessageDef_MiniTable(msg->msgdef), NULL, 0, Arena_get(msg->arena)); + upb_MessageDef_MiniTable(msg->msgdef), NULL, options, Arena_get(msg->arena)); if (status != kUpb_DecodeStatus_Ok) { rb_raise(cParseError, "Error occurred during parsing"); @@ -1043,24 +1065,43 @@ static VALUE Message_decode_json(int argc, VALUE* argv, VALUE klass) { /* * call-seq: - * MessageClass.encode(msg) => bytes + * MessageClass.encode(msg, options) => bytes * * Encodes the given message object to its serialized form in protocol buffers * wire format. + * @param options [Hash] options for the encoder + * recursion_limit: set to maximum encoding depth for message (default is 64) */ -static VALUE Message_encode(VALUE klass, VALUE msg_rb) { - Message* msg = ruby_to_Message(msg_rb); +static VALUE Message_encode(int argc, VALUE* argv, VALUE klass) { + Message* msg = ruby_to_Message(argv[0]); + int options = 0; const char* data; size_t size; - if (CLASS_OF(msg_rb) != klass) { + if (CLASS_OF(argv[0]) != klass) { rb_raise(rb_eArgError, "Message of wrong type."); } - upb_Arena* arena = upb_Arena_New(); + if (argc < 1 || argc > 2) { + rb_raise(rb_eArgError, "Expected 1 or 2 arguments."); + } - data = upb_Encode(msg->msg, upb_MessageDef_MiniTable(msg->msgdef), 0, arena, - &size); + if (argc == 2) { + VALUE hash_args = argv[1]; + if (TYPE(hash_args) != T_HASH) { + rb_raise(rb_eArgError, "Expected hash arguments."); + } + VALUE depth = rb_hash_lookup(hash_args, ID2SYM(rb_intern("recursion_limit"))); + + if (depth != Qnil && TYPE(depth) == T_FIXNUM) { + options |= UPB_DECODE_MAXDEPTH(FIX2INT(depth)); + } + } + + upb_Arena *arena = upb_Arena_New(); + + data = upb_Encode(msg->msg, upb_MessageDef_MiniTable(msg->msgdef), + options, arena, &size); if (data) { VALUE ret = rb_str_new(data, size); @@ -1186,8 +1227,8 @@ VALUE build_class_from_descriptor(VALUE descriptor) { rb_define_method(klass, "to_s", Message_inspect, 0); rb_define_method(klass, "[]", Message_index, 1); rb_define_method(klass, "[]=", Message_index_set, 2); - rb_define_singleton_method(klass, "decode", Message_decode, 1); - rb_define_singleton_method(klass, "encode", Message_encode, 1); + rb_define_singleton_method(klass, "decode", Message_decode, -1); + rb_define_singleton_method(klass, "encode", Message_encode, -1); rb_define_singleton_method(klass, "decode_json", Message_decode_json, -1); rb_define_singleton_method(klass, "encode_json", Message_encode_json, -1); rb_define_singleton_method(klass, "descriptor", Message_descriptor, 0); diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index f939a4c7dc..b7a6711051 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -59,16 +59,16 @@ require 'google/protobuf/repeated_field' module Google module Protobuf - def self.encode(msg) - msg.to_proto + def self.encode(msg, options = {}) + msg.to_proto(options) end def self.encode_json(msg, options = {}) msg.to_json(options) end - def self.decode(klass, proto) - klass.decode(proto) + def self.decode(klass, proto, options = {}) + klass.decode(proto, options) end def self.decode_json(klass, json, options = {}) diff --git a/ruby/lib/google/protobuf/message_exts.rb b/ruby/lib/google/protobuf/message_exts.rb index f432f89fed..6608521728 100644 --- a/ruby/lib/google/protobuf/message_exts.rb +++ b/ruby/lib/google/protobuf/message_exts.rb @@ -44,8 +44,8 @@ module Google self.class.encode_json(self, options) end - def to_proto - self.class.encode(self) + def to_proto(options = {}) + self.class.encode(self, options) end end diff --git a/ruby/src/main/java/com/google/protobuf/jruby/RubyMap.java b/ruby/src/main/java/com/google/protobuf/jruby/RubyMap.java index f7379b148e..7956eebe76 100644 --- a/ruby/src/main/java/com/google/protobuf/jruby/RubyMap.java +++ b/ruby/src/main/java/com/google/protobuf/jruby/RubyMap.java @@ -389,7 +389,7 @@ public class RubyMap extends RubyObject { return newMap; } - protected List build(ThreadContext context, RubyDescriptor descriptor, int depth) { + protected List build(ThreadContext context, RubyDescriptor descriptor, int depth, int recursionLimit) { List list = new ArrayList(); RubyClass rubyClass = (RubyClass) descriptor.msgclass(context); FieldDescriptor keyField = descriptor.getField("key"); @@ -398,7 +398,7 @@ public class RubyMap extends RubyObject { RubyMessage mapMessage = (RubyMessage) rubyClass.newInstance(context, Block.NULL_BLOCK); mapMessage.setField(context, keyField, key); mapMessage.setField(context, valueField, table.get(key)); - list.add(mapMessage.build(context, depth + 1)); + list.add(mapMessage.build(context, depth + 1, recursionLimit)); } return list; } diff --git a/ruby/src/main/java/com/google/protobuf/jruby/RubyMessage.java b/ruby/src/main/java/com/google/protobuf/jruby/RubyMessage.java index cf59f62597..b0a1f9010b 100644 --- a/ruby/src/main/java/com/google/protobuf/jruby/RubyMessage.java +++ b/ruby/src/main/java/com/google/protobuf/jruby/RubyMessage.java @@ -39,6 +39,7 @@ import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.Descriptors.FileDescriptor; import com.google.protobuf.Descriptors.OneofDescriptor; import com.google.protobuf.ByteString; +import com.google.protobuf.CodedInputStream; import com.google.protobuf.DynamicMessage; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; @@ -461,35 +462,63 @@ public class RubyMessage extends RubyObject { /* * call-seq: - * MessageClass.encode(msg) => bytes + * MessageClass.encode(msg, options = {}) => bytes * * Encodes the given message object to its serialized form in protocol buffers * wire format. + * @param options [Hash] options for the encoder + * recursion_limit: set to maximum encoding depth for message (default is 64) */ - @JRubyMethod(meta = true) - public static IRubyObject encode(ThreadContext context, IRubyObject recv, IRubyObject value) { - if (recv != value.getMetaClass()) { - throw context.runtime.newArgumentError("Tried to encode a " + value.getMetaClass() + " message with " + recv); + @JRubyMethod(required = 1, optional = 1, meta = true) + public static IRubyObject encode(ThreadContext context, IRubyObject recv, IRubyObject[] args) { + if (recv != args[0].getMetaClass()) { + throw context.runtime.newArgumentError("Tried to encode a " + args[0].getMetaClass() + " message with " + recv); } - RubyMessage message = (RubyMessage) value; - return context.runtime.newString(new ByteList(message.build(context).toByteArray())); + RubyMessage message = (RubyMessage) args[0]; + int recursionLimitInt = SINK_MAXIMUM_NESTING; + + if (args.length > 1) { + RubyHash options = (RubyHash) args[1]; + IRubyObject recursionLimit = options.fastARef(context.runtime.newSymbol("recursion_limit")); + + if (recursionLimit != null) { + recursionLimitInt = ((RubyNumeric) recursionLimit).getIntValue(); + } + } + return context.runtime.newString(new ByteList(message.build(context, 0, recursionLimitInt).toByteArray())); } /* * call-seq: - * MessageClass.decode(data) => message + * MessageClass.decode(data, options = {}) => message * * Decodes the given data (as a string containing bytes in protocol buffers wire * format) under the interpretation given by this message class's definition * and returns a message object with the corresponding field values. + * @param options [Hash] options for the decoder + * recursion_limit: set to maximum decoding depth for message (default is 100) */ - @JRubyMethod(meta = true) - public static IRubyObject decode(ThreadContext context, IRubyObject recv, IRubyObject data) { + @JRubyMethod(required = 1, optional = 1, meta = true) + public static IRubyObject decode(ThreadContext context, IRubyObject recv, IRubyObject[] args) { + IRubyObject data = args[0]; byte[] bin = data.convertToString().getBytes(); + CodedInputStream input = CodedInputStream.newInstance(bin); RubyMessage ret = (RubyMessage) ((RubyClass) recv).newInstance(context, Block.NULL_BLOCK); + + if (args.length == 2) { + if (!(args[1] instanceof RubyHash)) { + throw context.runtime.newArgumentError("Expected hash arguments."); + } + + IRubyObject recursionLimit = ((RubyHash) args[1]).fastARef(context.runtime.newSymbol("recursion_limit")); + if (recursionLimit != null) { + input.setRecursionLimit(((RubyNumeric) recursionLimit).getIntValue()); + } + } + try { - ret.builder.mergeFrom(bin); - } catch (InvalidProtocolBufferException e) { + ret.builder.mergeFrom(input); + } catch (Exception e) { throw RaiseException.from(context.runtime, (RubyClass) context.runtime.getClassFromPath("Google::Protobuf::ParseError"), e.getMessage()); } @@ -541,7 +570,7 @@ public class RubyMessage extends RubyObject { printer = printer.usingTypeRegistry(JsonFormat.TypeRegistry.newBuilder().add(message.descriptor).build()); try { - result = printer.print(message.build(context)); + result = printer.print(message.build(context, 0, SINK_MAXIMUM_NESTING)); } catch (InvalidProtocolBufferException e) { throw runtime.newRuntimeError(e.getMessage()); } catch (IllegalArgumentException e) { @@ -635,13 +664,9 @@ public class RubyMessage extends RubyObject { return ret; } - protected DynamicMessage build(ThreadContext context) { - return build(context, 0); - } - - protected DynamicMessage build(ThreadContext context, int depth) { - if (depth > SINK_MAXIMUM_NESTING) { - throw context.runtime.newRuntimeError("Maximum recursion depth exceeded during encoding."); + protected DynamicMessage build(ThreadContext context, int depth, int recursionLimit) { + if (depth >= recursionLimit) { + throw context.runtime.newRuntimeError("Recursion limit exceeded during encoding."); } // Handle the typical case where the fields.keySet contain the fieldDescriptors @@ -651,7 +676,7 @@ public class RubyMessage extends RubyObject { if (value instanceof RubyMap) { builder.clearField(fieldDescriptor); RubyDescriptor mapDescriptor = (RubyDescriptor) getDescriptorForField(context, fieldDescriptor); - for (DynamicMessage kv : ((RubyMap) value).build(context, mapDescriptor, depth)) { + for (DynamicMessage kv : ((RubyMap) value).build(context, mapDescriptor, depth, recursionLimit)) { builder.addRepeatedField(fieldDescriptor, kv); } @@ -660,7 +685,7 @@ public class RubyMessage extends RubyObject { builder.clearField(fieldDescriptor); for (int i = 0; i < repeatedField.size(); i++) { - Object item = convert(context, fieldDescriptor, repeatedField.get(i), depth, + Object item = convert(context, fieldDescriptor, repeatedField.get(i), depth, recursionLimit, /*isDefaultValueForBytes*/ false); builder.addRepeatedField(fieldDescriptor, item); } @@ -682,7 +707,7 @@ public class RubyMessage extends RubyObject { fieldDescriptor.getFullName().equals("google.protobuf.FieldDescriptorProto.default_value")) { isDefaultStringForBytes = true; } - builder.setField(fieldDescriptor, convert(context, fieldDescriptor, value, depth, isDefaultStringForBytes)); + builder.setField(fieldDescriptor, convert(context, fieldDescriptor, value, depth, recursionLimit, isDefaultStringForBytes)); } } @@ -702,7 +727,7 @@ public class RubyMessage extends RubyObject { builder.clearField(fieldDescriptor); RubyDescriptor mapDescriptor = (RubyDescriptor) getDescriptorForField(context, fieldDescriptor); - for (DynamicMessage kv : ((RubyMap) value).build(context, mapDescriptor, depth)) { + for (DynamicMessage kv : ((RubyMap) value).build(context, mapDescriptor, depth, recursionLimit)) { builder.addRepeatedField(fieldDescriptor, kv); } } @@ -814,7 +839,8 @@ public class RubyMessage extends RubyObject { // convert a ruby object to protobuf type, skip type check since it is checked on the way in private Object convert(ThreadContext context, FieldDescriptor fieldDescriptor, - IRubyObject value, int depth, boolean isDefaultStringForBytes) { + IRubyObject value, int depth, int recursionLimit, + boolean isDefaultStringForBytes) { Object val = null; switch (fieldDescriptor.getType()) { case INT32: @@ -855,7 +881,7 @@ public class RubyMessage extends RubyObject { } break; case MESSAGE: - val = ((RubyMessage) value).build(context, depth + 1); + val = ((RubyMessage) value).build(context, depth + 1, recursionLimit); break; case ENUM: EnumDescriptor enumDescriptor = fieldDescriptor.getEnumType(); @@ -1214,7 +1240,7 @@ public class RubyMessage extends RubyObject { private static final String CONST_SUFFIX = "_const"; private static final String HAS_PREFIX = "has_"; private static final String QUESTION_MARK = "?"; - private static final int SINK_MAXIMUM_NESTING = 63; + private static final int SINK_MAXIMUM_NESTING = 64; private Descriptor descriptor; private DynamicMessage.Builder builder; diff --git a/ruby/tests/basic.rb b/ruby/tests/basic.rb index b9d1554dbd..b74aa0f4d5 100755 --- a/ruby/tests/basic.rb +++ b/ruby/tests/basic.rb @@ -644,5 +644,19 @@ module BasicTest assert_equal 2, m.map_string_int32.size assert_equal 1, m.map_string_msg.size end + + def test_string_with_singleton_class_enabled + str = 'foobar' + # NOTE: Accessing a singleton class of an object changes its low level class representation + # as far as the C API's CLASS_OF() method concerned, exposing the issue + str.singleton_class + m = proto_module::TestMessage.new( + optional_string: str, + optional_bytes: str + ) + + assert_equal str, m.optional_string + assert_equal str, m.optional_bytes + end end end diff --git a/ruby/tests/common_tests.rb b/ruby/tests/common_tests.rb index 3ab9a0ca84..cd48b6916f 100644 --- a/ruby/tests/common_tests.rb +++ b/ruby/tests/common_tests.rb @@ -1245,7 +1245,7 @@ module CommonTests struct = struct_from_ruby(JSON.parse(json)) assert_equal json, struct.to_json - assert_raise(RuntimeError, "Maximum recursion depth exceeded during encoding") do + assert_raise(RuntimeError, "Recursion limit exceeded during encoding") do struct = Google::Protobuf::Struct.new struct.fields["foobar"] = Google::Protobuf::Value.new(struct_value: struct) Google::Protobuf::Struct.encode(struct) diff --git a/ruby/tests/encode_decode_test.rb b/ruby/tests/encode_decode_test.rb index 429ac43322..df2cd3632d 100755 --- a/ruby/tests/encode_decode_test.rb +++ b/ruby/tests/encode_decode_test.rb @@ -101,4 +101,55 @@ class EncodeDecodeTest < Test::Unit::TestCase assert_match json, "{\"CustomJsonName\":42}" end + def test_decode_depth_limit + msg = A::B::C::TestMessage.new( + optional_msg: A::B::C::TestMessage.new( + optional_msg: A::B::C::TestMessage.new( + optional_msg: A::B::C::TestMessage.new( + optional_msg: A::B::C::TestMessage.new( + optional_msg: A::B::C::TestMessage.new( + ) + ) + ) + ) + ) + ) + msg_encoded = A::B::C::TestMessage.encode(msg) + msg_out = A::B::C::TestMessage.decode(msg_encoded) + assert_match msg.to_json, msg_out.to_json + + assert_raise Google::Protobuf::ParseError do + A::B::C::TestMessage.decode(msg_encoded, { recursion_limit: 4 }) + end + + msg_out = A::B::C::TestMessage.decode(msg_encoded, { recursion_limit: 5 }) + assert_match msg.to_json, msg_out.to_json + end + + def test_encode_depth_limit + msg = A::B::C::TestMessage.new( + optional_msg: A::B::C::TestMessage.new( + optional_msg: A::B::C::TestMessage.new( + optional_msg: A::B::C::TestMessage.new( + optional_msg: A::B::C::TestMessage.new( + optional_msg: A::B::C::TestMessage.new( + ) + ) + ) + ) + ) + ) + msg_encoded = A::B::C::TestMessage.encode(msg) + msg_out = A::B::C::TestMessage.decode(msg_encoded) + assert_match msg.to_json, msg_out.to_json + + assert_raise RuntimeError do + A::B::C::TestMessage.encode(msg, { recursion_limit: 5 }) + end + + msg_encoded = A::B::C::TestMessage.encode(msg, { recursion_limit: 6 }) + msg_out = A::B::C::TestMessage.decode(msg_encoded) + assert_match msg.to_json, msg_out.to_json + end + end diff --git a/src/Makefile.am b/src/Makefile.am index ca9b81b238..d53cf57420 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -18,7 +18,7 @@ else PTHREAD_DEF = endif -PROTOBUF_VERSION = 28:3:0 +PROTOBUF_VERSION = 30:4:0 if GCC # Turn on all warnings except for sign comparison (we ignore sign comparison diff --git a/src/google/protobuf/any.pb.h b/src/google/protobuf/any.pb.h index b5c7ac91cb..79636ad91f 100644 --- a/src/google/protobuf/any.pb.h +++ b/src/google/protobuf/any.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3018000 +#if PROTOBUF_VERSION < 3019000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3018001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3019004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/any.proto b/src/google/protobuf/any.proto index 6ed8a23cf5..e2c2042fdc 100644 --- a/src/google/protobuf/any.proto +++ b/src/google/protobuf/any.proto @@ -64,7 +64,7 @@ option objc_class_prefix = "GPB"; // foo = any.unpack(Foo.class); // } // -// Example 3: Pack and unpack a message in Python. +// Example 3: Pack and unpack a message in Python. // // foo = Foo(...) // any = Any() @@ -74,7 +74,7 @@ option objc_class_prefix = "GPB"; // any.Unpack(foo) // ... // -// Example 4: Pack and unpack a message in Go +// Example 4: Pack and unpack a message in Go // // foo := &pb.Foo{...} // any, err := anypb.New(foo) @@ -95,7 +95,7 @@ option objc_class_prefix = "GPB"; // // // JSON -// ==== +// // The JSON representation of an `Any` value uses the regular // representation of the deserialized, embedded message, with an // additional field `@type` which contains the type URL. Example: diff --git a/src/google/protobuf/api.pb.h b/src/google/protobuf/api.pb.h index 48d41aacaa..4ffdbb2934 100644 --- a/src/google/protobuf/api.pb.h +++ b/src/google/protobuf/api.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3018000 +#if PROTOBUF_VERSION < 3019000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3018001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3019004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/arena.h b/src/google/protobuf/arena.h index 3165fc9d11..c3436d4119 100644 --- a/src/google/protobuf/arena.h +++ b/src/google/protobuf/arena.h @@ -37,9 +37,6 @@ #include #include #include -#ifdef max -#undef max // Visual Studio defines this macro -#endif #if defined(_MSC_VER) && !defined(_LIBCPP_STD_VER) && !_HAS_EXCEPTIONS // Work around bugs in MSVC header when _HAS_EXCEPTIONS=0. #include diff --git a/src/google/protobuf/arenastring.cc b/src/google/protobuf/arenastring.cc index 4ad82f6a3f..b358fdddab 100644 --- a/src/google/protobuf/arenastring.cc +++ b/src/google/protobuf/arenastring.cc @@ -57,6 +57,8 @@ namespace { // - alignof(std::string) #ifdef __STDCPP_DEFAULT_NEW_ALIGNMENT__ constexpr size_t kNewAlign = __STDCPP_DEFAULT_NEW_ALIGNMENT__; +#elif (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40900 +constexpr size_t kNewAlign = alignof(::max_align_t); #else constexpr size_t kNewAlign = alignof(std::max_align_t); #endif @@ -84,9 +86,18 @@ const std::string& LazyString::Init() const { namespace { +#if defined(NDEBUG) || !GOOGLE_PROTOBUF_INTERNAL_DONATE_STEAL + +class ScopedCheckPtrInvariants { + public: + explicit ScopedCheckPtrInvariants(const TaggedStringPtr*) {} +}; + +#endif // NDEBUG || !GOOGLE_PROTOBUF_INTERNAL_DONATE_STEAL + // Creates a heap allocated std::string value. -inline TaggedPtr CreateString(ConstStringParam value) { - TaggedPtr res; +inline TaggedStringPtr CreateString(ConstStringParam value) { + TaggedStringPtr res; res.SetAllocated(new std::string(value.data(), value.length())); return res; } @@ -94,8 +105,8 @@ inline TaggedPtr CreateString(ConstStringParam value) { #if !GOOGLE_PROTOBUF_INTERNAL_DONATE_STEAL // Creates an arena allocated std::string value. -TaggedPtr CreateArenaString(Arena& arena, ConstStringParam s) { - TaggedPtr res; +TaggedStringPtr CreateArenaString(Arena& arena, ConstStringParam s) { + TaggedStringPtr res; res.SetMutableArena(Arena::Create(&arena, s.data(), s.length())); return res; } @@ -104,13 +115,8 @@ TaggedPtr CreateArenaString(Arena& arena, ConstStringParam s) { } // namespace -std::string* ArenaStringPtr::SetAndReturnNewString() { - std::string* new_string = new std::string(); - tagged_ptr_.SetAllocated(new_string); - return new_string; -} - void ArenaStringPtr::Set(ConstStringParam value, Arena* arena) { + ScopedCheckPtrInvariants check(&tagged_ptr_); if (IsDefault()) { // If we're not on an arena, skip straight to a true string to avoid // possible copy cost later. @@ -122,6 +128,7 @@ void ArenaStringPtr::Set(ConstStringParam value, Arena* arena) { } void ArenaStringPtr::Set(std::string&& value, Arena* arena) { + ScopedCheckPtrInvariants check(&tagged_ptr_); if (IsDefault()) { NewString(arena, std::move(value)); } else if (IsFixedSizeArena()) { @@ -135,6 +142,7 @@ void ArenaStringPtr::Set(std::string&& value, Arena* arena) { } std::string* ArenaStringPtr::Mutable(Arena* arena) { + ScopedCheckPtrInvariants check(&tagged_ptr_); if (tagged_ptr_.IsMutable()) { return tagged_ptr_.Get(); } else { @@ -144,6 +152,7 @@ std::string* ArenaStringPtr::Mutable(Arena* arena) { std::string* ArenaStringPtr::Mutable(const LazyString& default_value, Arena* arena) { + ScopedCheckPtrInvariants check(&tagged_ptr_); if (tagged_ptr_.IsMutable()) { return tagged_ptr_.Get(); } else { @@ -152,6 +161,7 @@ std::string* ArenaStringPtr::Mutable(const LazyString& default_value, } std::string* ArenaStringPtr::MutableNoCopy(Arena* arena) { + ScopedCheckPtrInvariants check(&tagged_ptr_); if (tagged_ptr_.IsMutable()) { return tagged_ptr_.Get(); } else { @@ -169,6 +179,7 @@ std::string* ArenaStringPtr::MutableSlow(::google::protobuf::Arena* arena, } std::string* ArenaStringPtr::Release() { + ScopedCheckPtrInvariants check(&tagged_ptr_); if (IsDefault()) return nullptr; std::string* released = tagged_ptr_.Get(); @@ -181,6 +192,7 @@ std::string* ArenaStringPtr::Release() { } void ArenaStringPtr::SetAllocated(std::string* value, Arena* arena) { + ScopedCheckPtrInvariants check(&tagged_ptr_); // Release what we have first. Destroy(); @@ -206,6 +218,7 @@ void ArenaStringPtr::Destroy() { } void ArenaStringPtr::ClearToEmpty() { + ScopedCheckPtrInvariants check(&tagged_ptr_); if (IsDefault()) { // Already set to default -- do nothing. } else { @@ -220,6 +233,7 @@ void ArenaStringPtr::ClearToEmpty() { void ArenaStringPtr::ClearToDefault(const LazyString& default_value, ::google::protobuf::Arena* arena) { + ScopedCheckPtrInvariants check(&tagged_ptr_); (void)arena; if (IsDefault()) { // Already set to default -- do nothing. @@ -231,6 +245,7 @@ void ArenaStringPtr::ClearToDefault(const LazyString& default_value, const char* EpsCopyInputStream::ReadArenaString(const char* ptr, ArenaStringPtr* s, Arena* arena) { + ScopedCheckPtrInvariants check(&s->tagged_ptr_); GOOGLE_DCHECK(arena != nullptr); int size = ReadSize(&ptr); diff --git a/src/google/protobuf/arenastring.h b/src/google/protobuf/arenastring.h index a56ba6e056..a9f000d35e 100644 --- a/src/google/protobuf/arenastring.h +++ b/src/google/protobuf/arenastring.h @@ -94,8 +94,7 @@ class PROTOBUF_EXPORT LazyString { const std::string& Init() const; }; -template -class TaggedPtr { +class TaggedStringPtr { public: // Bit flags qualifying string properties. We can use up to 3 bits as // ptr_ is guaranteed and enforced to be aligned on 8 byte boundaries. @@ -130,30 +129,36 @@ class TaggedPtr { kFixedSizeArena = kArenaBit, }; - TaggedPtr() = default; - explicit constexpr TaggedPtr(ExplicitlyConstructedArenaString* ptr) + TaggedStringPtr() = default; + explicit constexpr TaggedStringPtr(ExplicitlyConstructedArenaString* ptr) : ptr_(ptr) {} // Sets the value to `p`, tagging the value as being a 'default' value. // See documentation for kDefault for more info. - inline const T* SetDefault(const T* p) { - return TagAs(kDefault, const_cast(p)); + inline const std::string* SetDefault(const std::string* p) { + return TagAs(kDefault, const_cast(p)); } // Sets the value to `p`, tagging the value as a heap allocated value. // Allocated strings are mutable and (as the name implies) owned. // `p` must not be null - inline T* SetAllocated(T* p) { return TagAs(kAllocated, p); } + inline std::string* SetAllocated(std::string* p) { + return TagAs(kAllocated, p); + } // Sets the value to `p`, tagging the value as a fixed size arena string. // See documentation for kFixedSizeArena for more info. // `p` must not be null - inline T* SetFixedSizeArena(T* p) { return TagAs(kFixedSizeArena, p); } + inline std::string* SetFixedSizeArena(std::string* p) { + return TagAs(kFixedSizeArena, p); + } // Sets the value to `p`, tagging the value as a mutable arena string. // See documentation for kMutableArena for more info. // `p` must not be null - inline T* SetMutableArena(T* p) { return TagAs(kMutableArena, p); } + inline std::string* SetMutableArena(std::string* p) { + return TagAs(kMutableArena, p); + } // Returns true if the contents of the current string are fully mutable. inline bool IsMutable() const { return as_int() & kMutableBit; } @@ -174,7 +179,9 @@ class TaggedPtr { } // Returns the contained string pointer. - inline T* Get() const { return reinterpret_cast(as_int() & ~kMask); } + inline std::string* Get() const { + return reinterpret_cast(as_int() & ~kMask); + } // Returns true if the contained pointer is null, indicating some error. // The Null value is only used during parsing for temporary values. @@ -186,7 +193,7 @@ class TaggedPtr { GOOGLE_DCHECK_EQ(reinterpret_cast(p) & kMask, 0UL); } - inline T* TagAs(Type type, T* p) { + inline std::string* TagAs(Type type, std::string* p) { GOOGLE_DCHECK(p != nullptr); assert_aligned(p); ptr_ = reinterpret_cast(reinterpret_cast(p) | type); @@ -197,8 +204,8 @@ class TaggedPtr { void* ptr_; }; -static_assert(std::is_trivial>::value, - "TaggedPtr must be trivial"); +static_assert(std::is_trivial::value, + "TaggedStringPtr must be trivial"); // This class encapsulates a pointer to a std::string with or without arena // owned contents, tagged by the bottom bits of the string pointer. It is a @@ -225,13 +232,6 @@ struct PROTOBUF_EXPORT ArenaStringPtr { ExplicitlyConstructedArenaString* default_value) : tagged_ptr_(default_value) {} - // Some methods below are overloaded on a `default_value` and on tags. - // The tagged overloads help reduce code size in the callers in generated - // code, while the `default_value` overloads are useful from reflection. - // By-value empty struct arguments are elided in the ABI. - struct EmptyDefault {}; - struct NonEmptyDefault {}; - // Called from generated code / reflection runtime only. Resets value to point // to a default string pointer, with the semantics that this ArenaStringPtr // does not own the pointed-to memory. Disregards initial value of ptr_ (so @@ -334,107 +334,9 @@ struct PROTOBUF_EXPORT ArenaStringPtr { ArenaStringPtr* lhs, Arena* lhs_arena); - // -------------------------------------------------------- - // Below functions will be removed in subsequent code change - // -------------------------------------------------------- -#ifdef DEPRECATED_METHODS_TO_BE_DELETED - PROTOBUF_NDEBUG_INLINE const std::string* GetPointer() const { - return UnsafeGetPointer(); - } - - template - void Set(DefaultArg, ConstStringParam value, Arena* arena) { - return Set(value, arena); - } - template - void Set(DefaultArg, std::string&& value, Arena* arena) { - return Set(std::move(value), arena); - } - template - void Set(DefaultArg, const char* s, Arena* arena) { - return Set(ConstStringParam{s}, arena); - } - template - void Set(DefaultArg, const char* s, size_t n, Arena* arena) { - return Set(ConstStringParam{s, n}, arena); - } - - void SetBytes(EmptyDefault, ConstStringParam value, Arena* arena) { - return Set(value, arena); - } - void SetBytes(NonEmptyDefault, ConstStringParam value, Arena* arena) { - return Set(value, arena); - } - void SetBytes(const std::string*, ConstStringParam value, Arena* arena) { - return Set(value, arena); - } - void SetBytes(EmptyDefault, std::string&& value, Arena* arena) { - return Set(std::move(value), arena); - } - void SetBytes(NonEmptyDefault, std::string&& value, Arena* arena) { - return Set(std::move(value), arena); - } - void SetBytes(const std::string*, std::string&& value, Arena* arena) { - return Set(std::move(value), arena); - } - void SetBytes(EmptyDefault, const char* s, Arena* arena) { - return Set(s, arena); - } - void SetBytes(NonEmptyDefault, const char* s, Arena* arena) { - return Set(s, arena); - } - void SetBytes(const std::string*, const char* s, Arena* arena) { - return Set(s, arena); - } - void SetBytes(EmptyDefault, const void* p, size_t n, Arena* arena) { - return SetBytes(p, n, arena); - } - void SetBytes(NonEmptyDefault, const void* p, size_t n, Arena* arena) { - return SetBytes(p, n, arena); - } - void SetBytes(const std::string*, const void* p, size_t n, Arena* arena) { - return SetBytes(p, n, arena); - } - - std::string* Mutable(EmptyDefault, Arena* arena) { return Mutable(arena); } - std::string* MutableNoArenaNoDefault(const std::string*) { - return Mutable(nullptr); - } - std::string* MutableNoCopy(const std::string*, ::google::protobuf::Arena* arena) { - return MutableNoCopy(arena); - } - - PROTOBUF_NODISCARD std::string* Release(const std::string*, Arena* arena) { - return Release(); - } - PROTOBUF_NODISCARD std::string* ReleaseNonDefault(const std::string*, - Arena* arena) { - return Release(); - } - - void SetAllocated(const std::string*, std::string* value, Arena* arena) { - SetAllocated(value, arena); - } - - void Destroy(const std::string*, ::google::protobuf::Arena* arena) { Destroy(); } - void Destroy(EmptyDefault, ::google::protobuf::Arena* arena) { Destroy(); } - void Destroy(NonEmptyDefault, ::google::protobuf::Arena* arena) { Destroy(); } - void DestroyNoArena(const std::string*) { Destroy(); } - - inline PROTOBUF_NDEBUG_INLINE static void InternalSwap(const std::string*, - ArenaStringPtr* rhs, - Arena* rhs_arena, - ArenaStringPtr* lhs, - Arena* lhs_arena) { - InternalSwap(rhs, rhs_arena, lhs, lhs_arena); - } -#endif // DEPRECATED_METHODS_TO_BE_DELETED - // Internal setter used only at parse time to directly set a donated string // value. - void UnsafeSetTaggedPointer(TaggedPtr value) { - tagged_ptr_ = value; - } + void UnsafeSetTaggedPointer(TaggedStringPtr value) { tagged_ptr_ = value; } // Generated code only! An optimization, in certain cases the generated // code is certain we can obtain a std::string with no default checks and // tag tests. @@ -455,7 +357,7 @@ struct PROTOBUF_EXPORT ArenaStringPtr { } } - TaggedPtr tagged_ptr_; + TaggedStringPtr tagged_ptr_; bool IsFixedSizeArena() const { return false; } @@ -472,18 +374,15 @@ struct PROTOBUF_EXPORT ArenaStringPtr { // Slow paths. // MutableSlow requires that !IsString() || IsDefault - // Variadic to support 0 args for EmptyDefault and 1 arg for LazyString. + // Variadic to support 0 args for empty default and 1 arg for LazyString. template std::string* MutableSlow(::google::protobuf::Arena* arena, const Lazy&... lazy_default); - // Sets value to a newly allocated string and returns it - std::string* SetAndReturnNewString(); - friend class EpsCopyInputStream; }; inline void ArenaStringPtr::InitDefault() { - tagged_ptr_ = TaggedPtr(&fixed_address_empty_string); + tagged_ptr_ = TaggedStringPtr(&fixed_address_empty_string); } inline void ArenaStringPtr::InitExternal(const std::string* str) { diff --git a/src/google/protobuf/arenastring_unittest.cc b/src/google/protobuf/arenastring_unittest.cc index bf91620ca3..0e5ea9b395 100644 --- a/src/google/protobuf/arenastring_unittest.cc +++ b/src/google/protobuf/arenastring_unittest.cc @@ -55,8 +55,6 @@ namespace protobuf { using internal::ArenaStringPtr; -using EmptyDefault = ArenaStringPtr::EmptyDefault; - const internal::LazyString nonempty_default{{{"default", 7}}, {nullptr}}; const std::string* empty_default = &internal::GetEmptyString(); diff --git a/src/google/protobuf/compiler/command_line_interface.cc b/src/google/protobuf/compiler/command_line_interface.cc index 3dff6c5cc7..a4b1f866ab 100644 --- a/src/google/protobuf/compiler/command_line_interface.cc +++ b/src/google/protobuf/compiler/command_line_interface.cc @@ -57,7 +57,7 @@ #include #include -#include //For PATH_MAX +#include // For PATH_MAX #include @@ -69,7 +69,6 @@ #include #include -#include #include #include #include @@ -82,6 +81,7 @@ #include #include #include +#include #include #include #include @@ -1676,8 +1676,9 @@ CommandLineInterface::InterpretArgument(const std::string& name, // On Windows, the shell (typically cmd.exe) does not expand wildcards in // file names (e.g. foo\*.proto), so we do it ourselves. switch (google::protobuf::io::win32::ExpandWildcards( - value, - [this](const string& path) { this->input_files_.push_back(path); })) { + value, [this](const std::string& path) { + this->input_files_.push_back(path); + })) { case google::protobuf::io::win32::ExpandWildcardsResult::kSuccess: break; case google::protobuf::io::win32::ExpandWildcardsResult:: @@ -2588,7 +2589,8 @@ void FormatFreeFieldNumbers(const std::string& name, StringAppendF(&output, " %d", next_free_number); } else { // Range - StringAppendF(&output, " %d-%d", next_free_number, i->first - 1); + StringAppendF(&output, " %d-%d", next_free_number, + i->first - 1); } } next_free_number = i->second; diff --git a/src/google/protobuf/compiler/cpp/cpp_enum_field.cc b/src/google/protobuf/compiler/cpp/cpp_enum_field.cc index 4df91c6ff5..89c864d111 100644 --- a/src/google/protobuf/compiler/cpp/cpp_enum_field.cc +++ b/src/google/protobuf/compiler/cpp/cpp_enum_field.cc @@ -130,13 +130,13 @@ void EnumFieldGenerator::GenerateSwappingCode(io::Printer* printer) const { void EnumFieldGenerator::GenerateConstructorCode(io::Printer* printer) const { Formatter format(printer, variables_); - format("$name$_ = $default$;\n"); + format("$field$ = $default$;\n"); } void EnumFieldGenerator::GenerateCopyConstructorCode( io::Printer* printer) const { Formatter format(printer, variables_); - format("$name$_ = from.$name$_;\n"); + format("$field$ = from.$field$;\n"); } void EnumFieldGenerator::GenerateSerializeWithCachedSizesToArray( @@ -216,7 +216,7 @@ void EnumOneofFieldGenerator::GenerateSwappingCode(io::Printer* printer) const { void EnumOneofFieldGenerator::GenerateConstructorCode( io::Printer* printer) const { Formatter format(printer, variables_); - format("$ns$::_$classname$_default_instance_.$name$_ = $default$;\n"); + format("$ns$::_$classname$_default_instance_.$field$ = $default$;\n"); } // =================================================================== diff --git a/src/google/protobuf/compiler/cpp/cpp_field.cc b/src/google/protobuf/compiler/cpp/cpp_field.cc index 0cf3d680da..52cc299db5 100644 --- a/src/google/protobuf/compiler/cpp/cpp_field.cc +++ b/src/google/protobuf/compiler/cpp/cpp_field.cc @@ -103,7 +103,7 @@ std::string GenerateTemplateForOneofString(const FieldDescriptor* descriptor, return strings::Substitute( StrCat("_internal_has_", field_name, "() ? ", field_pointer, " : ", default_value_pointer), - field_member, MakeDefaultName(descriptor)); + field_member, MakeDefaultFieldName(descriptor)); } std::string GenerateTemplateForSingleString(const FieldDescriptor* descriptor, @@ -115,7 +115,7 @@ std::string GenerateTemplateForSingleString(const FieldDescriptor* descriptor, if (descriptor->options().ctype() == google::protobuf::FieldOptions::STRING) { return strings::Substitute( "$0.IsDefault() ? &$1.get() : $0.UnsafeGetPointer()", field_member, - MakeDefaultName(descriptor)); + MakeDefaultFieldName(descriptor)); } return StrCat("&", field_member); @@ -150,7 +150,7 @@ void AddAccessorAnnotations(const FieldDescriptor* descriptor, google::protobuf::FileOptions::LITE_RUNTIME) { return; } - std::string field_member = (*variables)["field_member"]; + std::string field_member = (*variables)["field"]; const google::protobuf::OneofDescriptor* oneof_member = descriptor->real_containing_oneof(); const std::string proto_ns = (*variables)["proto_ns"]; @@ -241,11 +241,6 @@ void SetCommonFieldVariables(const FieldDescriptor* descriptor, (*variables)["number"] = StrCat(descriptor->number()); (*variables)["classname"] = ClassName(FieldScope(descriptor), false); (*variables)["declared_type"] = DeclaredTypeMethodName(descriptor->type()); - // TODO(b/218325252): convert all usages of "field_member" to "field" and - // remove this. The former may unnecessarily cause line breaks in protoc code. - // Note that the length of variables has no effect on the generated code. It - // only affects the readability of code template in protoc. - (*variables)["field_member"] = FieldMemberName(descriptor); (*variables)["field"] = FieldMemberName(descriptor); (*variables)["tag_size"] = StrCat( diff --git a/src/google/protobuf/compiler/cpp/cpp_file.cc b/src/google/protobuf/compiler/cpp/cpp_file.cc index dfce01a3ef..8b088e1180 100644 --- a/src/google/protobuf/compiler/cpp/cpp_file.cc +++ b/src/google/protobuf/compiler/cpp/cpp_file.cc @@ -511,9 +511,10 @@ void FileGenerator::GenerateSourceDefaultInstance(int idx, format( "PROTOBUF_ATTRIBUTE_INIT_PRIORITY2 std::true_type " "$1$::_init_inline_$2$_ = " - "($3$._instance.$2$_.Init(), std::true_type{});\n", + "($3$._instance.$4$.Init(), std::true_type{});\n", ClassName(generator->descriptor_), FieldName(field), - DefaultInstanceName(generator->descriptor_, options_)); + DefaultInstanceName(generator->descriptor_, options_), + FieldMemberName(field)); } } diff --git a/src/google/protobuf/compiler/cpp/cpp_helpers.h b/src/google/protobuf/compiler/cpp/cpp_helpers.h index c0217259ad..08a191a77a 100644 --- a/src/google/protobuf/compiler/cpp/cpp_helpers.h +++ b/src/google/protobuf/compiler/cpp/cpp_helpers.h @@ -486,6 +486,20 @@ inline std::string MakeDefaultName(const FieldDescriptor* field) { "_"; } +// Semantically distinct from MakeDefaultName in that it gives the C++ code +// referencing a default field from the message scope, rather than just the +// variable name. +// For example, declarations of default variables should always use just +// MakeDefaultName to produce code like: +// Type _i_give_permission_to_break_this_code_default_field_; +// +// Code that references these should use MakeDefaultFieldName, in case the field +// exists at some nested level like: +// internal_container_._i_give_permission_to_break_this_code_default_field_; +inline std::string MakeDefaultFieldName(const FieldDescriptor* field) { + return MakeDefaultName(field); +} + bool IsAnyMessage(const FileDescriptor* descriptor, const Options& options); bool IsAnyMessage(const Descriptor* descriptor, const Options& options); diff --git a/src/google/protobuf/compiler/cpp/cpp_map_field.cc b/src/google/protobuf/compiler/cpp/cpp_map_field.cc index 4c7765892a..5713fd8e8c 100644 --- a/src/google/protobuf/compiler/cpp/cpp_map_field.cc +++ b/src/google/protobuf/compiler/cpp/cpp_map_field.cc @@ -126,7 +126,7 @@ void MapFieldGenerator::GenerateInlineAccessorDefinitions( format( "inline const ::$proto_ns$::Map< $key_cpp$, $val_cpp$ >&\n" "$classname$::_internal_$name$() const {\n" - " return $name$_.GetMap();\n" + " return $field$.GetMap();\n" "}\n" "inline const ::$proto_ns$::Map< $key_cpp$, $val_cpp$ >&\n" "$classname$::$name$() const {\n" @@ -136,7 +136,7 @@ void MapFieldGenerator::GenerateInlineAccessorDefinitions( "}\n" "inline ::$proto_ns$::Map< $key_cpp$, $val_cpp$ >*\n" "$classname$::_internal_mutable_$name$() {\n" - " return $name$_.MutableMap();\n" + " return $field$.MutableMap();\n" "}\n" "inline ::$proto_ns$::Map< $key_cpp$, $val_cpp$ >*\n" "$classname$::mutable_$name$() {\n" @@ -148,17 +148,17 @@ void MapFieldGenerator::GenerateInlineAccessorDefinitions( void MapFieldGenerator::GenerateClearingCode(io::Printer* printer) const { Formatter format(printer, variables_); - format("$name$_.Clear();\n"); + format("$field$.Clear();\n"); } void MapFieldGenerator::GenerateMergingCode(io::Printer* printer) const { Formatter format(printer, variables_); - format("$name$_.MergeFrom(from.$name$_);\n"); + format("$field$.MergeFrom(from.$field$);\n"); } void MapFieldGenerator::GenerateSwappingCode(io::Printer* printer) const { Formatter format(printer, variables_); - format("$name$_.InternalSwap(&other->$name$_);\n"); + format("$field$.InternalSwap(&other->$field$);\n"); } void MapFieldGenerator::GenerateCopyConstructorCode( @@ -267,7 +267,7 @@ void MapFieldGenerator::GenerateIsInitialized(io::Printer* printer) const { Formatter format(printer, variables_); format( - "if (!::$proto_ns$::internal::AllAreInitialized($name$_)) return " + "if (!::$proto_ns$::internal::AllAreInitialized($field$)) return " "false;\n"); } @@ -285,7 +285,7 @@ void MapFieldGenerator::GenerateDestructorCode(io::Printer* printer) const { GOOGLE_CHECK(!IsFieldStripped(descriptor_, options_)); Formatter format(printer, variables_); - format("$name$_.Destruct();\n"); + format("$field$.Destruct();\n"); } void MapFieldGenerator::GenerateArenaDestructorCode( @@ -296,7 +296,7 @@ void MapFieldGenerator::GenerateArenaDestructorCode( Formatter format(printer, variables_); // _this is the object being destructed (we are inside a static method here). - format("_this->$name$_.Destruct();\n"); + format("_this->$field$.Destruct();\n"); } ArenaDtorNeeds MapFieldGenerator::NeedsArenaDestructor() const { diff --git a/src/google/protobuf/compiler/cpp/cpp_message.cc b/src/google/protobuf/compiler/cpp/cpp_message.cc index d16137787d..cdb8f112a0 100644 --- a/src/google/protobuf/compiler/cpp/cpp_message.cc +++ b/src/google/protobuf/compiler/cpp/cpp_message.cc @@ -1082,7 +1082,7 @@ void MessageGenerator::GenerateSingularFieldHasBits( // We maintain the invariant that for a submessage x, has_x() returning // true implies that x_ is not null. By giving this information to the // compiler, we allow it to eliminate unnecessary null checks later on. - format(" PROTOBUF_ASSUME(!value || $name$_ != nullptr);\n"); + format(" PROTOBUF_ASSUME(!value || $field$ != nullptr);\n"); } format( @@ -1097,13 +1097,13 @@ void MessageGenerator::GenerateSingularFieldHasBits( if (IsLazy(field, options_, scc_analyzer_)) { format( "inline bool $classname$::_internal_has_$name$() const {\n" - " return !$name$_.IsCleared();\n" + " return !$field$.IsCleared();\n" "}\n"); } else { format( "inline bool $classname$::_internal_has_$name$() const {\n" " return this != internal_default_instance() " - "&& $name$_ != nullptr;\n" + "&& $field$ != nullptr;\n" "}\n"); } format( @@ -2219,7 +2219,7 @@ std::pair MessageGenerator::GenerateOffsets( // Don't use the top bit because that is for unused fields. format("::_pbi::kInvalidFieldOffsetTag"); } else { - format("PROTOBUF_FIELD_OFFSET($classtype$, $1$_)", FieldName(field)); + format("PROTOBUF_FIELD_OFFSET($classtype$, $1$)", FieldMemberName(field)); } // Some information about a field is in the pdproto profile. The profile is @@ -2227,11 +2227,6 @@ std::pair MessageGenerator::GenerateOffsets( // offset of the field, so that the information is available when // reflectively accessing the field at run time. // - // Embed whether the field is used to the MSB of the offset. - if (!IsFieldUsed(field, options_)) { - format(" | 0x80000000u // unused\n"); - } - // Embed whether the field is eagerly verified lazy or inlined string to the // LSB of the offset. if (IsEagerlyVerifiedLazy(field, options_, scc_analyzer_)) { @@ -2420,16 +2415,16 @@ void MessageGenerator::GenerateConstructorBody(io::Printer* printer, std::string pod_template; if (copy_constructor) { pod_template = - "::memcpy(&$first$_, &from.$first$_,\n" - " static_cast(reinterpret_cast(&$last$_) -\n" - " reinterpret_cast(&$first$_)) + sizeof($last$_));\n"; + "::memcpy(&$first$, &from.$first$,\n" + " static_cast(reinterpret_cast(&$last$) -\n" + " reinterpret_cast(&$first$)) + sizeof($last$));\n"; } else { pod_template = "::memset(reinterpret_cast(this) + static_cast(\n" - " reinterpret_cast(&$first$_) - " + " reinterpret_cast(&$first$) - " "reinterpret_cast(this)),\n" - " 0, static_cast(reinterpret_cast(&$last$_) -\n" - " reinterpret_cast(&$first$_)) + sizeof($last$_));\n"; + " 0, static_cast(reinterpret_cast(&$last$) -\n" + " reinterpret_cast(&$first$)) + sizeof($last$));\n"; } for (int i = 0; i < optimized_order_.size(); ++i) { @@ -2445,9 +2440,9 @@ void MessageGenerator::GenerateConstructorBody(io::Printer* printer, if (it != runs.end() && it->second > 1) { // Use a memset, then skip run_length fields. const size_t run_length = it->second; - const std::string first_field_name = FieldName(field); + const std::string first_field_name = FieldMemberName(field); const std::string last_field_name = - FieldName(optimized_order_[i + run_length - 1]); + FieldMemberName(optimized_order_[i + run_length - 1]); format.Set("first", first_field_name); format.Set("last", last_field_name); @@ -2814,10 +2809,10 @@ void MessageGenerator::GenerateClear(io::Printer* printer) { .GenerateMessageClearingCode(printer); } else { format( - "::memset(&$1$_, 0, static_cast(\n" - " reinterpret_cast(&$2$_) -\n" - " reinterpret_cast(&$1$_)) + sizeof($2$_));\n", - FieldName(memset_start), FieldName(memset_end)); + "::memset(&$1$, 0, static_cast(\n" + " reinterpret_cast(&$2$) -\n" + " reinterpret_cast(&$1$)) + sizeof($2$));\n", + FieldMemberName(memset_start), FieldMemberName(memset_end)); } } @@ -2971,20 +2966,20 @@ void MessageGenerator::GenerateSwap(io::Printer* printer) { if (it != runs.end() && it->second > 1) { // Use a memswap, then skip run_length fields. const size_t run_length = it->second; - const std::string first_field_name = FieldName(field); + const std::string first_field_name = FieldMemberName(field); const std::string last_field_name = - FieldName(optimized_order_[i + run_length - 1]); + FieldMemberName(optimized_order_[i + run_length - 1]); format.Set("first", first_field_name); format.Set("last", last_field_name); format( "::PROTOBUF_NAMESPACE_ID::internal::memswap<\n" - " PROTOBUF_FIELD_OFFSET($classname$, $last$_)\n" - " + sizeof($classname$::$last$_)\n" - " - PROTOBUF_FIELD_OFFSET($classname$, $first$_)>(\n" - " reinterpret_cast(&$first$_),\n" - " reinterpret_cast(&other->$first$_));\n"); + " PROTOBUF_FIELD_OFFSET($classname$, $last$)\n" + " + sizeof($classname$::$last$)\n" + " - PROTOBUF_FIELD_OFFSET($classname$, $first$)>(\n" + " reinterpret_cast(&$first$),\n" + " reinterpret_cast(&other->$first$));\n"); i += run_length - 1; // ++i at the top of the loop. diff --git a/src/google/protobuf/compiler/cpp/cpp_message_field.cc b/src/google/protobuf/compiler/cpp/cpp_message_field.cc index 85b1066472..809a9e0442 100644 --- a/src/google/protobuf/compiler/cpp/cpp_message_field.cc +++ b/src/google/protobuf/compiler/cpp/cpp_message_field.cc @@ -61,10 +61,10 @@ void SetMessageVariables(const FieldDescriptor* descriptor, SetCommonFieldVariables(descriptor, variables, options); (*variables)["type"] = FieldMessageTypeName(descriptor, options); (*variables)["casted_member"] = ReinterpretCast( - (*variables)["type"] + "*", (*variables)["field_member"], implicit_weak); + (*variables)["type"] + "*", (*variables)["field"], implicit_weak); (*variables)["casted_member_const"] = ReinterpretCast("const " + (*variables)["type"] + "&", - "*" + (*variables)["field_member"], implicit_weak); + "*" + (*variables)["field"], implicit_weak); (*variables)["type_default_instance"] = QualifiedDefaultInstanceName(descriptor->message_type(), options); (*variables)["type_default_instance_ptr"] = ReinterpretCast( @@ -435,7 +435,7 @@ void MessageFieldGenerator::GenerateDestructorCode(io::Printer* printer) const { // care when handling them. format("if (this != internal_default_instance()) "); } - format("delete $name$_;\n"); + format("delete $field$;\n"); } void MessageFieldGenerator::GenerateConstructorCode( @@ -443,7 +443,7 @@ void MessageFieldGenerator::GenerateConstructorCode( GOOGLE_CHECK(!IsFieldStripped(descriptor_, options_)); Formatter format(printer, variables_); - format("$name$_ = nullptr;\n"); + format("$field$ = nullptr;\n"); } void MessageFieldGenerator::GenerateCopyConstructorCode( @@ -453,9 +453,9 @@ void MessageFieldGenerator::GenerateCopyConstructorCode( Formatter format(printer, variables_); format( "if (from._internal_has_$name$()) {\n" - " $name$_ = new $type$(*from.$name$_);\n" + " $field$ = new $type$(*from.$field$);\n" "} else {\n" - " $name$_ = nullptr;\n" + " $field$ = nullptr;\n" "}\n"); } diff --git a/src/google/protobuf/compiler/cpp/cpp_parse_function_generator.cc b/src/google/protobuf/compiler/cpp/cpp_parse_function_generator.cc index bd6af811dc..0983b5c07c 100644 --- a/src/google/protobuf/compiler/cpp/cpp_parse_function_generator.cc +++ b/src/google/protobuf/compiler/cpp/cpp_parse_function_generator.cc @@ -73,10 +73,6 @@ int TagSize(uint32_t field_number) { return 2; } -const char* CodedTagType(int tag_size) { - return tag_size == 1 ? "uint8_t" : "uint16_t"; -} - std::string FieldParseFunctionName( const TailCallTableInfo::FieldEntryInfo& entry, const Options& options); @@ -91,11 +87,12 @@ bool IsFieldEligibleForFastParsing( IsLazy(field, options, scc_analyzer)) { return false; } - switch (field->type()) { - // Groups are not handled on the fast path. - case FieldDescriptor::TYPE_GROUP: - return false; + // We will check for a valid auxiliary index range later. However, we might + // want to change the value we check for inlined string fields. + int aux_idx = entry.aux_idx; + + switch (field->type()) { case FieldDescriptor::TYPE_ENUM: // If enum values are not validated at parse time, then this field can be // handled on the fast path like an int32. @@ -110,11 +107,16 @@ bool IsFieldEligibleForFastParsing( // Some bytes fields can be handled on fast path. case FieldDescriptor::TYPE_STRING: case FieldDescriptor::TYPE_BYTES: - if (field->options().ctype() != FieldOptions::STRING || - !field->default_value_string().empty() || - IsStringInlined(field, options)) { + if (field->options().ctype() != FieldOptions::STRING) { return false; } + if (IsStringInlined(field, options)) { + GOOGLE_CHECK(!field->is_repeated()); + // For inlined strings, the donation state index is stored in the + // `aux_idx` field of the fast parsing info. We need to check the range + // of that value instead of the auxiliary index. + aux_idx = entry.inlined_string_idx; + } break; default: @@ -130,7 +132,7 @@ bool IsFieldEligibleForFastParsing( // If the field needs auxiliary data, then the aux index is needed. This // must fit in a uint8_t. - if (entry.aux_idx > std::numeric_limits::max()) { + if (aux_idx > std::numeric_limits::max()) { return false; } @@ -191,7 +193,12 @@ std::vector SplitFastFieldsForSize( // If this field does not have presence, then it can set an out-of-bounds // bit (tailcall parsing uses a uint64_t for hasbits, but only stores 32). info.hasbit_idx = HasHasbit(field) ? entry.hasbit_idx : 63; - info.aux_idx = static_cast(entry.aux_idx); + if (IsStringInlined(field, options)) { + GOOGLE_CHECK(!field->is_repeated()); + info.aux_idx = static_cast(entry.inlined_string_idx); + } else { + info.aux_idx = static_cast(entry.aux_idx); + } } return result; } @@ -248,6 +255,7 @@ std::vector FilterMiniParsedFields( break; case FieldDescriptor::TYPE_MESSAGE: + case FieldDescriptor::TYPE_GROUP: // TODO(b/210762816): support remaining field types. if (field->is_map() || IsWeak(field, options) || IsImplicitWeakField(field, options, scc_analyzer) || @@ -273,7 +281,9 @@ std::vector FilterMiniParsedFields( TailCallTableInfo::TailCallTableInfo( const Descriptor* descriptor, const Options& options, const std::vector& ordered_fields, - const std::vector& has_bit_indices, MessageSCCAnalyzer* scc_analyzer) { + const std::vector& has_bit_indices, + const std::vector& inlined_string_indices, + MessageSCCAnalyzer* scc_analyzer) { int oneof_count = descriptor->real_oneof_decl_count(); // If this message has any oneof fields, store the case offset in the first // auxiliary entry. @@ -284,6 +294,22 @@ TailCallTableInfo::TailCallTableInfo( aux_entries.push_back(StrCat( "_fl::Offset{offsetof(", ClassName(descriptor), ", _oneof_case_)}")); } + + int inlined_string_count = 0; + for (const FieldDescriptor* field : ordered_fields) { + if (IsString(field, options) && IsStringInlined(field, options)) { + ++inlined_string_count; + } + } + // If this message has any inlined string fields, store the donation state + // offset in the second auxiliary entry. + if (inlined_string_count > 0) { + aux_entries.resize(2); // pad if necessary + aux_entries[1] = + StrCat("_fl::Offset{offsetof(", ClassName(descriptor), + ", _inlined_string_donated_)}"); + } + // Fill in mini table entries. for (const FieldDescriptor* field : ordered_fields) { field_entries.push_back( @@ -326,7 +352,7 @@ TailCallTableInfo::TailCallTableInfo( const EnumDescriptor* enum_type = field->enum_type(); GOOGLE_CHECK_GT(enum_type->value_count(), 0) << enum_type->DebugString(); - // Check if the enum values are a single, continguous range. + // Check if the enum values are a single, contiguous range. std::vector enum_values; for (int i = 0, N = enum_type->value_count(); i < N; ++i) { enum_values.push_back(enum_type->value(i)->number()); @@ -348,6 +374,19 @@ TailCallTableInfo::TailCallTableInfo( aux_entries.push_back( StrCat(QualifiedClassName(enum_type, options), "_IsValid")); } + } else if ((field->type() == FieldDescriptor::TYPE_STRING || + field->type() == FieldDescriptor::TYPE_BYTES) && + IsStringInlined(field, options)) { + GOOGLE_CHECK(!field->is_repeated()); + // Inlined strings have an extra marker to represent their donation state. + int idx = inlined_string_indices[field->index()]; + // For mini parsing, the donation state index is stored as an `offset` + // auxiliary entry. + entry.aux_idx = aux_entries.size(); + aux_entries.push_back(StrCat("_fl::Offset{", idx, "}")); + // For fast table parsing, the donation state index is stored instead of + // the aux_idx (this will limit the range to 8 bits). + entry.inlined_string_idx = idx; } } @@ -407,7 +446,8 @@ ParseFunctionGenerator::ParseFunctionGenerator( num_hasbits_(max_has_bit_index) { if (should_generate_tctable()) { tc_table_info_.reset(new TailCallTableInfo( - descriptor_, options_, ordered_fields_, has_bit_indices, scc_analyzer)); + descriptor_, options_, ordered_fields_, has_bit_indices, + inlined_string_indices, scc_analyzer)); } SetCommonVars(options_, &variables_); SetUnknownFieldsVariable(descriptor_, options_, &variables_); @@ -532,6 +572,31 @@ void ParseFunctionGenerator::GenerateTailcallFallbackFunction( "}\n"); } +struct SkipEntry16 { + uint16_t skipmap; + uint16_t field_entry_offset; +}; +struct SkipEntryBlock { + uint32_t first_fnum; + std::vector entries; +}; +struct NumToEntryTable { + uint32_t skipmap32; // for fields #1 - #32 + std::vector blocks; + // Compute the number of uint16_t required to represent this table. + int size16() const { + int size = 2; // for the termination field# + for (const auto& block : blocks) { + // 2 for the field#, 1 for a count of skip entries, 2 for each entry. + size += 3 + block.entries.size() * 2; + } + return size; + } +}; + +static NumToEntryTable MakeNumToEntryTable( + const std::vector& field_descriptors); + void ParseFunctionGenerator::GenerateDataDecls(io::Printer* printer) { if (!should_generate_tctable()) { return; @@ -542,11 +607,13 @@ void ParseFunctionGenerator::GenerateDataDecls(io::Printer* printer) { format("#ifdef PROTOBUF_TAIL_CALL_TABLE_PARSER_ENABLED\n"); format.Indent(); } + auto field_num_to_entry_table = MakeNumToEntryTable(ordered_fields_); format( - "static const ::$proto_ns$::internal::TcParseTable<$1$, $2$, $3$, $4$> " - "_table_;\n", + "static const ::$proto_ns$::internal::" + "TcParseTable<$1$, $2$, $3$, $4$, $5$> _table_;\n", tc_table_info_->table_size_log2, ordered_fields_.size(), - tc_table_info_->aux_entries.size(), CalculateFieldNamesSize()); + tc_table_info_->aux_entries.size(), CalculateFieldNamesSize(), + field_num_to_entry_table.size16()); if (should_generate_guarded_tctable()) { format.Outdent(); format("#endif // PROTOBUF_TAIL_CALL_TABLE_PARSER_ENABLED\n"); @@ -614,6 +681,68 @@ void ParseFunctionGenerator::GenerateLoopingParseFunction(Formatter& format) { "}\n"); } +static NumToEntryTable MakeNumToEntryTable( + const std::vector& field_descriptors) { + NumToEntryTable num_to_entry_table; + num_to_entry_table.skipmap32 = static_cast(-1); + + // skip_entry_block is the current block of SkipEntries that we're + // appending to. cur_block_first_fnum is the number of the first + // field represented by the block. + uint16_t field_entry_index = 0; + uint16_t N = field_descriptors.size(); + // First, handle field numbers 1-32, which affect only the initial + // skipmap32 and don't generate additional skip-entry blocks. + for (; field_entry_index != N; ++field_entry_index) { + auto* field_descriptor = field_descriptors[field_entry_index]; + if (field_descriptor->number() > 32) break; + auto skipmap32_index = field_descriptor->number() - 1; + num_to_entry_table.skipmap32 -= 1 << skipmap32_index; + } + // If all the field numbers were less than or equal to 32, we will have + // no further entries to process, and we are already done. + if (field_entry_index == N) return num_to_entry_table; + + SkipEntryBlock* block = nullptr; + bool start_new_block = true; + // To determine sparseness, track the field number corresponding to + // the start of the most recent skip entry. + uint32_t last_skip_entry_start = 0; + for (; field_entry_index != N; ++field_entry_index) { + auto* field_descriptor = field_descriptors[field_entry_index]; + uint32_t fnum = field_descriptor->number(); + GOOGLE_CHECK_GT(fnum, last_skip_entry_start); + if (start_new_block == false) { + // If the next field number is within 15 of the last_skip_entry_start, we + // continue writing just to that entry. If it's between 16 and 31 more, + // then we just extend the current block by one. If it's more than 31 + // more, we have to add empty skip entries in order to continue using the + // existing block. Obviously it's just 32 more, it doesn't make sense to + // start a whole new block, since new blocks mean having to write out + // their starting field number, which is 32 bits, as well as the size of + // the additional block, which is 16... while an empty SkipEntry16 only + // costs 32 bits. So if it was 48 more, it's a slight space win; we save + // 16 bits, but probably at the cost of slower run time. We're choosing + // 96 for now. + if (fnum - last_skip_entry_start > 96) start_new_block = true; + } + if (start_new_block) { + num_to_entry_table.blocks.push_back(SkipEntryBlock{fnum}); + block = &num_to_entry_table.blocks.back(); + start_new_block = false; + } + + auto skip_entry_num = (fnum - block->first_fnum) / 16; + auto skip_entry_index = (fnum - block->first_fnum) % 16; + while (skip_entry_num >= block->entries.size()) + block->entries.push_back({0xFFFF, field_entry_index}); + block->entries[skip_entry_num].skipmap -= 1 << (skip_entry_index); + + last_skip_entry_start = fnum - skip_entry_index; + } + return num_to_entry_table; +} + void ParseFunctionGenerator::GenerateTailCallTable(Formatter& format) { GOOGLE_CHECK(should_generate_tctable()); // All entries without a fast-path parsing function need a fallback. @@ -635,12 +764,15 @@ void ParseFunctionGenerator::GenerateTailCallTable(Formatter& format) { // maps, weak fields, lazy, more than 1 extension range. In the cases // the table is sufficient we can use a generic routine, that just handles // unknown fields and potentially an extension range. + auto field_num_to_entry_table = MakeNumToEntryTable(ordered_fields_); format( "PROTOBUF_ATTRIBUTE_INIT_PRIORITY1\n" - "const ::_pbi::TcParseTable<$1$, $2$, $3$, $4$> $classname$::_table_ = " + "const ::_pbi::TcParseTable<$1$, $2$, $3$, $4$, $5$> " + "$classname$::_table_ = " "{\n", tc_table_info_->table_size_log2, ordered_fields_.size(), - tc_table_info_->aux_entries.size(), CalculateFieldNamesSize()); + tc_table_info_->aux_entries.size(), CalculateFieldNamesSize(), + field_num_to_entry_table.size16()); { auto table_scope = format.ScopedIndent(); format("{\n"); @@ -663,28 +795,16 @@ void ParseFunctionGenerator::GenerateTailCallTable(Formatter& format) { format("$1$, $2$, // max_field_number, fast_idx_mask\n", (ordered_fields_.empty() ? 0 : ordered_fields_.back()->number()), (((1 << tc_table_info_->table_size_log2) - 1) << 3)); - - // Determine the sequential fields that can be looked up by index: - uint16_t num_sequential_fields = 0; - uint16_t sequential_fields_start = 0; - if (!ordered_fields_.empty() && - ordered_fields_.front()->number() <= - std::numeric_limits::max()) { - sequential_fields_start = ordered_fields_[0]->number(); - const FieldDescriptor* previous_field = ordered_fields_[0]; - const int N = std::min(ordered_fields_.size(), - size_t{std::numeric_limits::max()} + 1); - for (int i = 1; i < N; ++i) { - const FieldDescriptor* current_field = ordered_fields_[i]; - if (current_field->number() > previous_field->number() + 1) { - break; - } - ++num_sequential_fields; - previous_field = current_field; - } + format( + "offsetof(decltype(_table_), field_lookup_table),\n" + "$1$, // skipmap\n", + field_num_to_entry_table.skipmap32); + if (ordered_fields_.empty()) { + format( + "offsetof(decltype(_table_), field_names), // no field_entries\n"); + } else { + format("offsetof(decltype(_table_), field_entries),\n"); } - format("$1$, $2$, // num_sequential_fields, sequential_fields_start\n", - num_sequential_fields, sequential_fields_start); format( "$1$, // num_field_entries\n" @@ -708,32 +828,41 @@ void ParseFunctionGenerator::GenerateTailCallTable(Formatter& format) { auto fast_scope = format.ScopedIndent(); GenerateFastFieldEntries(format); } + format("}}, {{\n"); + { + // field_lookup_table[] + auto field_lookup_scope = format.ScopedIndent(); + int line_entries = 0; + for (int i = 0, N = field_num_to_entry_table.blocks.size(); i < N; ++i) { + SkipEntryBlock& entry_block = field_num_to_entry_table.blocks[i]; + format("$1$, $2$, $3$,\n", entry_block.first_fnum & 65535, + entry_block.first_fnum / 65536, entry_block.entries.size()); + for (auto se16 : entry_block.entries) { + if (line_entries == 0) { + format("$1$, $2$,", se16.skipmap, se16.field_entry_offset); + ++line_entries; + } else if (line_entries < 5) { + format(" $1$, $2$,", se16.skipmap, se16.field_entry_offset); + ++line_entries; + } else { + format(" $1$, $2$,\n", se16.skipmap, se16.field_entry_offset); + line_entries = 0; + } + } + } + if (line_entries) format("\n"); + format("65535, 65535\n"); + } if (ordered_fields_.empty()) { GOOGLE_LOG_IF(DFATAL, !tc_table_info_->aux_entries.empty()) << "Invalid message: " << descriptor_->full_name() << " has " << tc_table_info_->aux_entries.size() << " auxiliary field entries, but no fields"; - format("}},\n" - "// no field_numbers, field_entries, or aux_entries\n" - "{{\n"); + format( + "}},\n" + "// no field_entries, or aux_entries\n" + "{{\n"); } else { - format("}}, {{\n"); - { - // field_numbers[] - auto field_number_scope = format.ScopedIndent(); - for (int i = 0, N = ordered_fields_.size(); i < N; ++i) { - const FieldDescriptor* field = ordered_fields_[i]; - if (i > 0) { - if (i % 10 == 0) { - format(",\n"); - } else { - format(", "); - } - } - format("$1$", field->number()); - } - format("\n"); - } format("}}, {{\n"); { // field_entries[] @@ -759,7 +888,7 @@ void ParseFunctionGenerator::GenerateTailCallTable(Formatter& format) { } // ordered_fields_.empty() { // field_names[] - auto field_scope = format.ScopedIndent(); + auto field_name_scope = format.ScopedIndent(); GenerateFieldNames(format); } format("}},\n"); @@ -995,12 +1124,6 @@ void ParseFunctionGenerator::GenerateArenaString(Formatter& format, if (HasHasbit(field)) { format("_Internal::set_has_$1$(&$has_bits$);\n", FieldName(field)); } - std::string default_string = - field->default_value_string().empty() - ? "::" + ProtobufNamespace(options_) + - "::internal::GetEmptyStringAlreadyInited()" - : QualifiedClassName(field->containing_type(), options_) + - "::" + MakeDefaultName(field) + ".get()"; format( "if (arena != nullptr) {\n" " ptr = ctx->ReadArenaString(ptr, &$msg$$field$, arena"); @@ -1008,13 +1131,8 @@ void ParseFunctionGenerator::GenerateArenaString(Formatter& format, GOOGLE_DCHECK(!inlined_string_indices_.empty()); int inlined_string_index = inlined_string_indices_[field->index()]; GOOGLE_DCHECK_GT(inlined_string_index, 0); - format( - ", $msg$_internal_$name$_donated()" - ", &$msg$_inlined_string_donated_[$1$]" - ", ~0x$2$u" - ", $this$", - inlined_string_index / 32, - strings::Hex(1u << (inlined_string_index % 32), strings::ZERO_PAD_8)); + format(", &$msg$_inlined_string_donated_[0], $1$, $this$", + inlined_string_index); } else { GOOGLE_DCHECK(field->default_value_string().empty()); } @@ -1309,7 +1427,7 @@ void ParseFunctionGenerator::GenerateFieldBody( format("_Internal::set_has_$name$(&$has_bits$);\n"); } format( - "$msg$$name$_ = " + "$msg$$field$ = " "::$proto_ns$::internal::UnalignedLoad<$primitive_type$>(ptr);\n" "ptr += sizeof($primitive_type$);\n"); } @@ -1437,7 +1555,6 @@ void ParseFunctionGenerator::GenerateFieldSwitch( format.Indent(); for (const auto* field : fields) { - // Set abbreviated form instead of field_member. format.Set("field", FieldMemberName(field)); PrintFieldComment(format, field); format("case $1$:\n", field->number()); @@ -1543,6 +1660,9 @@ std::string FieldParseFunctionName( case FieldDescriptor::TYPE_BYTES: name.append("B"); + if (IsStringInlined(field, options)) { + name.append("i"); + } break; case FieldDescriptor::TYPE_STRING: switch (GetUtf8CheckMode(field, options)) { @@ -1560,11 +1680,17 @@ std::string FieldParseFunctionName( << static_cast(GetUtf8CheckMode(field, options)); return ""; } + if (IsStringInlined(field, options)) { + name.append("i"); + } break; case FieldDescriptor::TYPE_MESSAGE: name.append("M"); break; + case FieldDescriptor::TYPE_GROUP: + name.append("G"); + break; default: GOOGLE_LOG(DFATAL) << "Type not handled: " << field->DebugString(); diff --git a/src/google/protobuf/compiler/cpp/cpp_parse_function_generator.h b/src/google/protobuf/compiler/cpp/cpp_parse_function_generator.h index a1621fd2d8..d98f085e30 100644 --- a/src/google/protobuf/compiler/cpp/cpp_parse_function_generator.h +++ b/src/google/protobuf/compiler/cpp/cpp_parse_function_generator.h @@ -51,6 +51,7 @@ struct TailCallTableInfo { TailCallTableInfo(const Descriptor* descriptor, const Options& options, const std::vector& ordered_fields, const std::vector& has_bit_indices, + const std::vector& inlined_string_indices, MessageSCCAnalyzer* scc_analyzer); // Fields parsed by the table fast-path. @@ -67,6 +68,7 @@ struct TailCallTableInfo { struct FieldEntryInfo { const FieldDescriptor* field; int hasbit_idx; + int inlined_string_idx; uint16_t aux_idx; // True for enums entirely covered by the start/length fields of FieldAux: bool is_enum_range; diff --git a/src/google/protobuf/compiler/cpp/cpp_primitive_field.cc b/src/google/protobuf/compiler/cpp/cpp_primitive_field.cc index f307256a03..53661d6e34 100644 --- a/src/google/protobuf/compiler/cpp/cpp_primitive_field.cc +++ b/src/google/protobuf/compiler/cpp/cpp_primitive_field.cc @@ -186,13 +186,13 @@ void PrimitiveFieldGenerator::GenerateSwappingCode(io::Printer* printer) const { void PrimitiveFieldGenerator::GenerateConstructorCode( io::Printer* printer) const { Formatter format(printer, variables_); - format("$name$_ = $default$;\n"); + format("$field$ = $default$;\n"); } void PrimitiveFieldGenerator::GenerateCopyConstructorCode( io::Printer* printer) const { Formatter format(printer, variables_); - format("$name$_ = from.$name$_;\n"); + format("$field$ = from.$field$;\n"); } void PrimitiveFieldGenerator::GenerateSerializeWithCachedSizesToArray( @@ -286,7 +286,7 @@ void PrimitiveOneofFieldGenerator::GenerateSwappingCode( void PrimitiveOneofFieldGenerator::GenerateConstructorCode( io::Printer* printer) const { Formatter format(printer, variables_); - format("$ns$::_$classname$_default_instance_.$name$_ = $default$;\n"); + format("$ns$::_$classname$_default_instance_.$field$ = $default$;\n"); } // =================================================================== diff --git a/src/google/protobuf/compiler/cpp/cpp_string_field.cc b/src/google/protobuf/compiler/cpp/cpp_string_field.cc index 9a57271c32..fa28c629c5 100644 --- a/src/google/protobuf/compiler/cpp/cpp_string_field.cc +++ b/src/google/protobuf/compiler/cpp/cpp_string_field.cc @@ -58,8 +58,8 @@ void SetStringVariables(const FieldDescriptor* descriptor, (*variables)["default"] = DefaultValue(options, descriptor); (*variables)["default_length"] = StrCat(descriptor->default_value_string().length()); - std::string default_variable_string = MakeDefaultName(descriptor); - (*variables)["default_variable_name"] = default_variable_string; + (*variables)["default_variable_name"] = MakeDefaultName(descriptor); + (*variables)["default_variable_field"] = MakeDefaultFieldName(descriptor); if (descriptor->default_value_string().empty()) { (*variables)["default_string"] = kNS + "GetEmptyStringAlreadyInited()"; @@ -67,8 +67,8 @@ void SetStringVariables(const FieldDescriptor* descriptor, (*variables)["lazy_variable_args"] = ""; } else { (*variables)["lazy_variable"] = - QualifiedClassName(descriptor->containing_type(), options) + - "::" + default_variable_string; + StrCat(QualifiedClassName(descriptor->containing_type(), options), + "::", MakeDefaultFieldName(descriptor)); (*variables)["default_string"] = (*variables)["lazy_variable"] + ".get()"; (*variables)["default_value"] = "nullptr"; @@ -205,7 +205,7 @@ void StringFieldGenerator::GenerateInlineAccessorDefinitions( if (!descriptor_->default_value_string().empty()) { format( " if ($field$.IsDefault()) return " - "$default_variable_name$.get();\n"); + "$default_variable_field$.get();\n"); } format( " return _internal_$name$();\n" @@ -345,7 +345,7 @@ void StringFieldGenerator::GenerateNonInlineAccessorDefinitions( if (!descriptor_->default_value_string().empty()) { format( "const ::$proto_ns$::internal::LazyString " - "$classname$::$default_variable_name$" + "$classname$::$default_variable_field$" "{{{$default$, $default_length$}}, {nullptr}};\n"); } } @@ -430,12 +430,12 @@ void StringFieldGenerator::GenerateConstructorCode(io::Printer* printer) const { return; } GOOGLE_DCHECK(!inlined_); - format("$name$_.InitDefault();\n"); + format("$field$.InitDefault();\n"); if (IsString(descriptor_, options_) && descriptor_->default_value_string().empty()) { format( "#ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING\n" - " $name$_.Set(\"\", GetArenaForAllocation());\n" + " $field$.Set(\"\", GetArenaForAllocation());\n" "#endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING\n"); } } @@ -445,7 +445,7 @@ void StringFieldGenerator::GenerateCopyConstructorCode( Formatter format(printer, variables_); GenerateConstructorCode(printer); if (inlined_) { - format("new (&$name$_) ::$proto_ns$::internal::InlinedStringField();\n"); + format("new (&$field$) ::_pbi::InlinedStringField();\n"); } if (HasHasbit(descriptor_)) { diff --git a/src/google/protobuf/compiler/java/java_field.h b/src/google/protobuf/compiler/java/java_field.h index a7c995c4ef..6ac01b99ff 100644 --- a/src/google/protobuf/compiler/java/java_field.h +++ b/src/google/protobuf/compiler/java/java_field.h @@ -156,7 +156,7 @@ template <> FieldGeneratorMap::~FieldGeneratorMap(); -// Field information used in FieldGeneartors. +// Field information used in FieldGenerators. struct FieldGeneratorInfo { std::string name; std::string capitalized_name; diff --git a/src/google/protobuf/compiler/java/java_generator.cc b/src/google/protobuf/compiler/java/java_generator.cc index 2ee08744ed..f75f8b3165 100644 --- a/src/google/protobuf/compiler/java/java_generator.cc +++ b/src/google/protobuf/compiler/java/java_generator.cc @@ -39,6 +39,7 @@ #include #include +#include #include #include #include diff --git a/src/google/protobuf/compiler/java/java_helpers.cc b/src/google/protobuf/compiler/java/java_helpers.cc index c66da524e1..3d9dc4001d 100644 --- a/src/google/protobuf/compiler/java/java_helpers.cc +++ b/src/google/protobuf/compiler/java/java_helpers.cc @@ -40,9 +40,9 @@ #include #include -#include #include #include +#include #include #include #include @@ -66,15 +66,27 @@ namespace { const char* kDefaultPackage = ""; -// Names that should be avoided as field names. -// Using them will cause the compiler to generate accessors whose names are -// colliding with methods defined in base classes. +// Names that should be avoided (in UpperCamelCase format). +// Using them will cause the compiler to generate accessors whose names +// collide with methods defined in base classes. +// Keep this list in sync with specialFieldNames in +// java/core/src/main/java/com/google/protobuf/DescriptorMessageInfoFactory.java const char* kForbiddenWordList[] = { - // message base class: - "cached_size", - "serialized_size", // java.lang.Object: - "class", + "Class", + // com.google.protobuf.MessageLiteOrBuilder: + "DefaultInstanceForType", + // com.google.protobuf.MessageLite: + "ParserForType", + "SerializedSize", + // com.google.protobuf.MessageOrBuilder: + "AllFields", + "DescriptorForType", + "InitializationErrorString", + // TODO(b/219045204): re-enable + // "UnknownFields", + // obsolete. kept for backwards compatibility of generated code + "CachedSize", }; const std::unordered_set* kReservedNames = @@ -93,7 +105,7 @@ const std::unordered_set* kReservedNames = bool IsForbidden(const std::string& field_name) { for (int i = 0; i < GOOGLE_ARRAYSIZE(kForbiddenWordList); ++i) { - if (field_name == kForbiddenWordList[i]) { + if (UnderscoresToCamelCase(field_name, true) == kForbiddenWordList[i]) { return true; } } diff --git a/src/google/protobuf/compiler/main.cc b/src/google/protobuf/compiler/main.cc index 7bddb314ba..08d77a639c 100644 --- a/src/google/protobuf/compiler/main.cc +++ b/src/google/protobuf/compiler/main.cc @@ -77,7 +77,7 @@ int ProtobufMain(int argc, char* argv[]) { python::Generator py_generator; cli.RegisterGenerator("--python_out", "--python_opt", &py_generator, "Generate Python source file."); - // Pyton pyi + // Python pyi python::PyiGenerator pyi_generator; cli.RegisterGenerator("--pyi_out", &pyi_generator, "Generate python pyi stub."); diff --git a/src/google/protobuf/compiler/php/php_generator.cc b/src/google/protobuf/compiler/php/php_generator.cc index 6b8ee524aa..b87dd5ef4f 100644 --- a/src/google/protobuf/compiler/php/php_generator.cc +++ b/src/google/protobuf/compiler/php/php_generator.cc @@ -48,28 +48,29 @@ const std::string kDescriptorMetadataFile = const std::string kDescriptorDirName = "Google/Protobuf/Internal"; const std::string kDescriptorPackageName = "Google\\Protobuf\\Internal"; const char* const kReservedNames[] = { - "abstract", "and", "array", "as", "break", - "callable", "case", "catch", "class", "clone", - "const", "continue", "declare", "default", "die", - "do", "echo", "else", "elseif", "empty", - "enddeclare", "endfor", "endforeach", "endif", "endswitch", - "endwhile", "eval", "exit", "extends", "final", - "finally", "fn", "for", "foreach", "function", - "global", "goto", "if", "implements", "include", - "include_once", "instanceof", "insteadof", "interface", "isset", - "list", "match", "namespace", "new", "or", - "print", "private", "protected", "public", "require", - "require_once", "return", "static", "switch", "throw", - "trait", "try", "unset", "use", "var", - "while", "xor", "yield", "int", "float", - "bool", "string", "true", "false", "null", - "void", "iterable"}; + "abstract", "and", "array", "as", "break", + "callable", "case", "catch", "class", "clone", + "const", "continue", "declare", "default", "die", + "do", "echo", "else", "elseif", "empty", + "enddeclare", "endfor", "endforeach", "endif", "endswitch", + "endwhile", "eval", "exit", "extends", "final", + "finally", "fn", "for", "foreach", "function", + "global", "goto", "if", "implements", "include", + "include_once", "instanceof", "insteadof", "interface", "isset", + "list", "match", "namespace", "new", "or", + "parent", "print", "private", "protected", "public", + "require", "require_once", "return", "self", "static", + "switch", "throw", "trait", "try", "unset", + "use", "var", "while", "xor", "yield", + "int", "float", "bool", "string", "true", + "false", "null", "void", "iterable"}; const char* const kValidConstantNames[] = { "int", "float", "bool", "string", "true", - "false", "null", "void", "iterable", + "false", "null", "void", "iterable", "parent", + "self" }; -const int kReservedNamesSize = 77; -const int kValidConstantNamesSize = 9; +const int kReservedNamesSize = 79; +const int kValidConstantNamesSize = 11; const int kFieldSetter = 1; const int kFieldGetter = 2; const int kFieldProperty = 3; @@ -1114,7 +1115,7 @@ void GenerateAddFilesToPool(const FileDescriptor* file, const Options& options, std::map dependency_count; std::set nodes_without_dependency; FileDescriptorSet sorted_file_set; - + AnalyzeDependencyForFile( file, &nodes_without_dependency, &deps, &dependency_count); @@ -2065,7 +2066,7 @@ void GenerateCMessage(const Descriptor* message, io::Printer* printer) { break; default: break; - } + } printer->Print( " ZEND_FE_END\n" diff --git a/src/google/protobuf/compiler/plugin.pb.h b/src/google/protobuf/compiler/plugin.pb.h index 4f7f75a99a..889c550fb2 100644 --- a/src/google/protobuf/compiler/plugin.pb.h +++ b/src/google/protobuf/compiler/plugin.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3018000 +#if PROTOBUF_VERSION < 3019000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3018001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3019004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/compiler/python/python_generator.cc b/src/google/protobuf/compiler/python/python_generator.cc index 7d9e7eae67..8f70925a70 100644 --- a/src/google/protobuf/compiler/python/python_generator.cc +++ b/src/google/protobuf/compiler/python/python_generator.cc @@ -54,13 +54,13 @@ #include #include -#include #include #include #include #include #include #include +#include #include #include @@ -1284,7 +1284,7 @@ void Generator::FixOptionsForEnum(const EnumDescriptor& enum_descriptor) const { if (value_options != "None") { PrintDescriptorOptionsFixingCode( StringPrintf("%s.values_by_name[\"%s\"]", descriptor_name.c_str(), - value_descriptor.name().c_str()), + value_descriptor.name().c_str()), value_options, printer_); } } diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 8504f888b8..5dd319035e 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -412,6 +412,10 @@ class FlatAllocatorImpl { const auto push_name = [&](std::string new_name) { for (size_t i = 0; i < names.size(); ++i) { + // Do not compare the full_name. It is unlikely to match, except in + // custom json_name. We are not taking this into account in + // PlanFieldNames so better to not try it. + if (i == 1) continue; if (names[i] == new_name) return i; } names.push_back(std::move(new_name)); diff --git a/src/google/protobuf/descriptor.pb.h b/src/google/protobuf/descriptor.pb.h index f7db21c87b..4765badff2 100644 --- a/src/google/protobuf/descriptor.pb.h +++ b/src/google/protobuf/descriptor.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3018000 +#if PROTOBUF_VERSION < 3019000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3018001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3019004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/descriptor.proto b/src/google/protobuf/descriptor.proto index e5605a9b3b..49ec6534b1 100644 --- a/src/google/protobuf/descriptor.proto +++ b/src/google/protobuf/descriptor.proto @@ -813,8 +813,8 @@ message SourceCodeInfo { // location. // // Each element is a field number or an index. They form a path from - // the root FileDescriptorProto to the place where the definition. For - // example, this path: + // the root FileDescriptorProto to the place where the definition occurs. + // For example, this path: // [ 4, 3, 2, 7, 1 ] // refers to: // file.message_type(3) // 4, 3 diff --git a/src/google/protobuf/descriptor_unittest.cc b/src/google/protobuf/descriptor_unittest.cc index 479d8f171e..2f57cd70e9 100644 --- a/src/google/protobuf/descriptor_unittest.cc +++ b/src/google/protobuf/descriptor_unittest.cc @@ -865,6 +865,22 @@ TEST_F(DescriptorTest, FieldNamesDedup) { ElementsAre("fieldname7")); } +TEST_F(DescriptorTest, FieldNameDedupJsonEqFull) { + // Test a regression where json_name == full_name + FileDescriptorProto proto; + proto.set_name("file"); + auto* message = AddMessage(&proto, "Name1"); + auto* field = + AddField(message, "Name2", 1, FieldDescriptorProto::LABEL_OPTIONAL, + FieldDescriptorProto::TYPE_INT32); + field->set_json_name("Name1.Name2"); + auto* file = pool_.BuildFile(proto); + EXPECT_EQ(file->message_type(0)->name(), "Name1"); + EXPECT_EQ(file->message_type(0)->field(0)->name(), "Name2"); + EXPECT_EQ(file->message_type(0)->field(0)->full_name(), "Name1.Name2"); + EXPECT_EQ(file->message_type(0)->field(0)->json_name(), "Name1.Name2"); +} + TEST_F(DescriptorTest, FieldsByIndex) { ASSERT_EQ(4, message_->field_count()); EXPECT_EQ(foo_, message_->field(0)); diff --git a/src/google/protobuf/duration.pb.h b/src/google/protobuf/duration.pb.h index 9e879145c0..816f670a0f 100644 --- a/src/google/protobuf/duration.pb.h +++ b/src/google/protobuf/duration.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3018000 +#if PROTOBUF_VERSION < 3019000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3018001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3019004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/empty.pb.h b/src/google/protobuf/empty.pb.h index 8b54992a64..668f740650 100644 --- a/src/google/protobuf/empty.pb.h +++ b/src/google/protobuf/empty.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3018000 +#if PROTOBUF_VERSION < 3019000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3018001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3019004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/field_mask.pb.h b/src/google/protobuf/field_mask.pb.h index a7a55b4522..355373cc60 100644 --- a/src/google/protobuf/field_mask.pb.h +++ b/src/google/protobuf/field_mask.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3018000 +#if PROTOBUF_VERSION < 3019000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3018001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3019004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index dad11bd238..c63f3ad724 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -75,7 +75,6 @@ using google::protobuf::internal::RepeatedPtrFieldBase; using google::protobuf::internal::StringSpaceUsedExcludingSelfLong; using google::protobuf::internal::WrappedMutex; - namespace google { namespace protobuf { diff --git a/src/google/protobuf/generated_message_reflection.h b/src/google/protobuf/generated_message_reflection.h index c35302922a..68af374b5c 100644 --- a/src/google/protobuf/generated_message_reflection.h +++ b/src/google/protobuf/generated_message_reflection.h @@ -227,13 +227,6 @@ struct ReflectionSchema { return false; } - // Returns true if the field's accessor is called by any external code (aka, - // non proto library code). - bool IsFieldUsed(const FieldDescriptor* field) const { - (void)field; - return true; - } - bool IsFieldStripped(const FieldDescriptor* field) const { (void)field; return false; @@ -271,9 +264,9 @@ struct ReflectionSchema { if (type == FieldDescriptor::TYPE_MESSAGE || type == FieldDescriptor::TYPE_STRING || type == FieldDescriptor::TYPE_BYTES) { - return v & 0x7FFFFFFEu; + return v & 0xFFFFFFFEu; } - return v & 0x7FFFFFFFu; + return v; } static bool Inlined(uint32_t v, FieldDescriptor::Type type) { diff --git a/src/google/protobuf/generated_message_tctable_decl.h b/src/google/protobuf/generated_message_tctable_decl.h index d1e15f6a20..b1bb1def70 100644 --- a/src/google/protobuf/generated_message_tctable_decl.h +++ b/src/google/protobuf/generated_message_tctable_decl.h @@ -129,8 +129,9 @@ struct alignas(uint64_t) TcParseTableBase { uint32_t extension_range_high; uint32_t max_field_number; uint8_t fast_idx_mask; - uint8_t num_sequential_fields; - uint16_t sequential_fields_start; + uint16_t lookup_table_offset; + uint32_t skipmap32; + uint32_t field_entries_offset; uint16_t num_field_entries; uint16_t num_aux_entries; @@ -150,8 +151,9 @@ struct alignas(uint64_t) TcParseTableBase { uint16_t has_bits_offset, uint16_t extension_offset, uint32_t extension_range_low, uint32_t extension_range_high, uint32_t max_field_number, uint8_t fast_idx_mask, - uint8_t num_sequential_fields, uint16_t sequential_fields_start, - uint16_t num_field_entries, uint16_t num_aux_entries, uint32_t aux_offset, + uint16_t lookup_table_offset, uint32_t skipmap32, + uint32_t field_entries_offset, uint16_t num_field_entries, + uint16_t num_aux_entries, uint32_t aux_offset, const MessageLite* default_instance, TailCallParseFunc fallback) : has_bits_offset(has_bits_offset), extension_offset(extension_offset), @@ -159,8 +161,9 @@ struct alignas(uint64_t) TcParseTableBase { extension_range_high(extension_range_high), max_field_number(max_field_number), fast_idx_mask(fast_idx_mask), - num_sequential_fields(num_sequential_fields), - sequential_fields_start(sequential_fields_start), + lookup_table_offset(lookup_table_offset), + skipmap32(skipmap32), + field_entries_offset(field_entries_offset), num_field_entries(num_field_entries), num_aux_entries(num_aux_entries), aux_offset(aux_offset), @@ -179,16 +182,10 @@ struct alignas(uint64_t) TcParseTableBase { return reinterpret_cast(this + 1) + idx; } - // Returns a begin/end iterator (pointer) for the field numbers array. - // The field numbers are a parallel array to the `FieldEntry` array. Note that - // not all numbers may be valid fields; in these cases, the corresponding - // field entry will have a field kind of `field_layout::kFkNone`. - const uint32_t* field_numbers_begin() const { - return reinterpret_cast( - fast_entry((fast_idx_mask >> 3) + 1)); - } - const uint32_t* field_numbers_end() const { - return field_numbers_begin() + num_field_entries; + // Returns a begin iterator (pointer) to the start of the field lookup table. + const uint16_t* field_lookup_begin() const { + return reinterpret_cast(reinterpret_cast(this) + + lookup_table_offset); } // Field entry for all fields. @@ -201,7 +198,8 @@ struct alignas(uint64_t) TcParseTableBase { // Returns a begin iterator (pointer) to the start of the field entries array. const FieldEntry* field_entries_begin() const { - return reinterpret_cast(field_numbers_end()); + return reinterpret_cast( + reinterpret_cast(this) + field_entries_offset); } // Auxiliary entries for field types that need extra information. @@ -248,7 +246,8 @@ static_assert(sizeof(TcParseTableBase::FieldEntry) <= 16, "Field entry is too big."); template + size_t kNumFieldAux = 0, size_t kNameTableSize = 0, + size_t kFieldLookupSize = 2> struct TcParseTable { TcParseTableBase header; @@ -261,8 +260,9 @@ struct TcParseTable { std::array fast_entries; + // Just big enough to find all the field entries. + std::array field_lookup_table; // Entries for all fields: - std::array field_numbers; std::array field_entries; std::array aux_entries; std::array field_names; @@ -273,24 +273,26 @@ struct TcParseTable { // However, different implementations have different sizeof(std::array). // Skipping the member makes offset computations portable. template -struct TcParseTable { + size_t kNameTableSize, size_t kFieldLookupSize> +struct TcParseTable { TcParseTableBase header; std::array fast_entries; - std::array field_numbers; + std::array field_lookup_table; std::array field_entries; std::array field_names; }; // Partial specialization: if there are no fields at all, then we can save space // by skipping the field numbers and entries. -template -struct TcParseTable<0, 0, 0, kNameTableSize> { +template +struct TcParseTable<0, 0, 0, kNameTableSize, kFieldLookupSize> { TcParseTableBase header; // N.B.: the fast entries are sized by log2, so 2**0 fields = 1 entry. // The fast parsing loop will always use this entry, so it must be present. std::array fast_entries; + std::array field_lookup_table; std::array field_names; }; diff --git a/src/google/protobuf/generated_message_tctable_impl.h b/src/google/protobuf/generated_message_tctable_impl.h index 5c1eec2cd4..a8ffad448d 100644 --- a/src/google/protobuf/generated_message_tctable_impl.h +++ b/src/google/protobuf/generated_message_tctable_impl.h @@ -339,6 +339,7 @@ class PROTOBUF_EXPORT TcParser final { // Functions referenced by generated fast tables (string types): // B: bytes S: string U: UTF-8 string + // (empty): ArenaStringPtr i: InlinedString // S: singular R: repeated // 1/2: tag length (bytes) static const char* FastBS1(PROTOBUF_TC_PARAM_DECL); @@ -354,14 +355,25 @@ class PROTOBUF_EXPORT TcParser final { static const char* FastUR1(PROTOBUF_TC_PARAM_DECL); static const char* FastUR2(PROTOBUF_TC_PARAM_DECL); + static const char* FastBiS1(PROTOBUF_TC_PARAM_DECL); + static const char* FastBiS2(PROTOBUF_TC_PARAM_DECL); + static const char* FastSiS1(PROTOBUF_TC_PARAM_DECL); + static const char* FastSiS2(PROTOBUF_TC_PARAM_DECL); + static const char* FastUiS1(PROTOBUF_TC_PARAM_DECL); + static const char* FastUiS2(PROTOBUF_TC_PARAM_DECL); + // Functions referenced by generated fast tables (message types): - // M: message + // M: message G: group // S: singular R: repeated // 1/2: tag length (bytes) static const char* FastMS1(PROTOBUF_TC_PARAM_DECL); static const char* FastMS2(PROTOBUF_TC_PARAM_DECL); static const char* FastMR1(PROTOBUF_TC_PARAM_DECL); static const char* FastMR2(PROTOBUF_TC_PARAM_DECL); + static const char* FastGS1(PROTOBUF_TC_PARAM_DECL); + static const char* FastGS2(PROTOBUF_TC_PARAM_DECL); + static const char* FastGR1(PROTOBUF_TC_PARAM_DECL); + static const char* FastGR2(PROTOBUF_TC_PARAM_DECL); template static inline T& RefAt(void* x, size_t offset) { @@ -402,9 +414,9 @@ class PROTOBUF_EXPORT TcParser final { private: friend class GeneratedTcTableLiteTest; - template + template static inline const char* SingularParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL); - template + template static inline const char* RepeatedParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL); static inline PROTOBUF_ALWAYS_INLINE void SyncHasbits( @@ -496,7 +508,7 @@ class PROTOBUF_EXPORT TcParser final { MessageLite* msg); // UTF-8 validation: - static void ReportFastUtf8Error(uint16_t coded_tag, + static void ReportFastUtf8Error(uint32_t decoded_tag, const TcParseTableBase* table); static bool MpVerifyUtf8(StringPiece wire_bytes, const TcParseTableBase* table, diff --git a/src/google/protobuf/generated_message_tctable_lite.cc b/src/google/protobuf/generated_message_tctable_lite.cc index e0ba5fc4a1..b3861a16f7 100644 --- a/src/google/protobuf/generated_message_tctable_lite.cc +++ b/src/google/protobuf/generated_message_tctable_lite.cc @@ -46,8 +46,6 @@ namespace google { namespace protobuf { namespace internal { -const uint32_t TcParser::kMtSmallScanSize; - using FieldEntry = TcParseTableBase::FieldEntry; ////////////////////////////////////////////////////////////////////////////// @@ -139,62 +137,121 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::Error( return nullptr; } +// On the fast path, a (matching) 1-byte tag already has the decoded value. +static uint32_t FastDecodeTag(uint8_t coded_tag) { + return coded_tag; +} + +// On the fast path, a (matching) 2-byte tag always needs to be decoded. +static uint32_t FastDecodeTag(uint16_t coded_tag) { + uint32_t result = coded_tag; + result += static_cast(coded_tag); + return result >> 1; +} + ////////////////////////////////////////////////////////////////////////////// // Core mini parsing implementation: ////////////////////////////////////////////////////////////////////////////// +// Field lookup table layout: +// +// Because it consists of a series of variable-length segments, the lookuup +// table is organized within an array of uint16_t, and each element is either +// a uint16_t or a uint32_t stored little-endian as a pair of uint16_t. +// +// Its fundamental building block maps 16 contiguously ascending field numbers +// to their locations within the field entry table: + +struct SkipEntry16 { + uint16_t skipmap; + uint16_t field_entry_offset; +}; + +// The skipmap is a bitfield of which of those field numbers do NOT have a +// field entry. The lowest bit of the skipmap corresponds to the lowest of +// the 16 field numbers, so if a proto had only fields 1, 2, 3, and 7, the +// skipmap would contain 0b11111111'10111000. +// +// The field lookup table begins with a single 32-bit skipmap that maps the +// field numbers 1 through 32. This is because the majority of proto +// messages only contain fields numbered 1 to 32. +// +// The rest of the lookup table is a repeated series of +// { 32-bit field #, #SkipEntry16s, {SkipEntry16...} } +// That is, the next thing is a pair of uint16_t that form the next +// lowest field number that the lookup table handles. If this number is -1, +// that is the end of the table. Then there is a uint16_t that is +// the number of contiguous SkipEntry16 entries that follow, and then of +// course the SkipEntry16s themselves. + +// Originally developed and tested at https://godbolt.org/z/vbc7enYcf + // Returns the address of the field for `tag` in the table's field entries. // Returns nullptr if the field was not found. const TcParseTableBase::FieldEntry* TcParser::FindFieldEntry( const TcParseTableBase* table, uint32_t field_num) { const FieldEntry* const field_entries = table->field_entries_begin(); - // Most messages have fields numbered sequentially. If the decoded tag is - // within that range, we can look up the field by index. - const uint32_t sequential_start = table->sequential_fields_start; - uint32_t adjusted_field_num = field_num - sequential_start; - const uint32_t num_sequential = table->num_sequential_fields; - if (PROTOBUF_PREDICT_TRUE(adjusted_field_num < num_sequential)) { - return field_entries + adjusted_field_num; - } - - // Check if this field is larger than the max in the table. This is often an - // extension. - if (field_num > table->max_field_number) { - return nullptr; - } - - // Otherwise, scan the next few field numbers, skipping the first - // `num_sequential` entries. - const uint32_t* const field_num_begin = table->field_numbers_begin(); - const uint32_t small_scan_limit = - std::min(num_sequential + kMtSmallScanSize, - static_cast(table->num_field_entries)); - for (uint32_t i = num_sequential; i < small_scan_limit; ++i) { - if (field_num <= field_num_begin[i]) { - if (PROTOBUF_PREDICT_FALSE(field_num != field_num_begin[i])) { - // Field number mismatch. - return nullptr; - } - return field_entries + i; + uint32_t fstart = 1; + uint32_t adj_fnum = field_num - fstart; + + if (PROTOBUF_PREDICT_TRUE(adj_fnum < 32)) { + uint32_t skipmap = table->skipmap32; + uint32_t skipbit = 1 << adj_fnum; + if (PROTOBUF_PREDICT_FALSE(skipmap & skipbit)) return nullptr; + skipmap &= skipbit - 1; +#if (__GNUC__ || __clang__) && __POPCNT__ + // Note: here and below, skipmap typically has very few set bits + // (31 in the worst case, but usually zero) so a loop isn't that + // bad, and a compiler-generated popcount is typically only + // worthwhile if the processor itself has hardware popcount support. + adj_fnum -= __builtin_popcount(skipmap); +#else + while (skipmap) { + --adj_fnum; + skipmap &= skipmap - 1; } +#endif + return field_entries + adj_fnum; } - - // Finally, look up with binary search. - const uint32_t* const field_num_end = table->field_numbers_end(); - auto it = std::lower_bound(field_num_begin + small_scan_limit, field_num_end, - field_num); - if (it == field_num_end) { - // The only reason for binary search failing is if there was nothing to - // search. - GOOGLE_DCHECK_EQ(field_num_begin + small_scan_limit, field_num_end) << field_num; - return nullptr; - } - if (PROTOBUF_PREDICT_FALSE(*it != field_num)) { - // Field number mismatch. - return nullptr; + const uint16_t* lookup_table = table->field_lookup_begin(); + for (;;) { +#ifdef PROTOBUF_LITTLE_ENDIAN + memcpy(&fstart, lookup_table, sizeof(fstart)); +#else + fstart = lookup_table[0] | (lookup_table[1] << 16); +#endif + lookup_table += sizeof(fstart) / sizeof(*lookup_table); + uint32_t num_skip_entries = *lookup_table++; + if (field_num < fstart) return nullptr; + adj_fnum = field_num - fstart; + uint32_t skip_num = adj_fnum / 16; + if (PROTOBUF_PREDICT_TRUE(skip_num < num_skip_entries)) { + // for each group of 16 fields we have: + // a bitmap of 16 bits + // a 16-bit field-entry offset for the first of them. + auto* skip_data = lookup_table + (adj_fnum / 16) * (sizeof(SkipEntry16) / + sizeof(uint16_t)); + SkipEntry16 se = {skip_data[0], skip_data[1]}; + adj_fnum &= 15; + uint32_t skipmap = se.skipmap; + uint16_t skipbit = 1 << adj_fnum; + if (PROTOBUF_PREDICT_FALSE(skipmap & skipbit)) return nullptr; + skipmap &= skipbit - 1; + adj_fnum += se.field_entry_offset; +#if (__GNUC__ || __clang__) && __POPCNT__ + adj_fnum -= __builtin_popcount(skipmap); +#else + while (skipmap) { + --adj_fnum; + skipmap &= skipmap - 1; + } +#endif + return field_entries + adj_fnum; + } + lookup_table += + num_skip_entries * (sizeof(SkipEntry16) / sizeof(*lookup_table)); } - return field_entries + (it - field_num_begin); } // Field names are stored in a format of: @@ -300,12 +357,13 @@ inline PROTOBUF_ALWAYS_INLINE void InvertPacked(TcFieldData& data) { // Message fields ////////////////////////////////////////////////////////////////////////////// -template +template inline PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); } + auto saved_tag = UnalignedLoad(ptr); ptr += sizeof(TagType); hasbits |= (uint64_t{1} << data.hasbit_idx()); auto& field = RefAt(msg, data.offset()); @@ -315,25 +373,39 @@ const char* TcParser::SingularParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL) { field = default_instance->New(ctx->data().arena); } SyncHasbits(msg, hasbits, table); + if (group_coding) { + return ctx->ParseGroup(field, ptr, FastDecodeTag(saved_tag)); + } return ctx->ParseMessage(field, ptr); } const char* TcParser::FastMS1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } const char* TcParser::FastMS2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +const char* TcParser::FastGS1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +const char* TcParser::FastGS2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } -template +template inline PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); } + auto saved_tag = UnalignedLoad(ptr); ptr += sizeof(TagType); SyncHasbits(msg, hasbits, table); const MessageLite* default_instance = @@ -341,16 +413,29 @@ const char* TcParser::RepeatedParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL) { auto& field = RefAt(msg, data.offset()); MessageLite* submsg = field.Add>(default_instance); + if (group_coding) { + return ctx->ParseGroup(submsg, ptr, FastDecodeTag(saved_tag)); + } return ctx->ParseMessage(submsg, ptr); } const char* TcParser::FastMR1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } const char* TcParser::FastMR2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +const char* TcParser::FastGR1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +const char* TcParser::FastGR2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } @@ -972,12 +1057,9 @@ void PrintUTF8ErrorLog(StringPiece message_name, StringPiece field_name, const char* operation_str, bool emit_stacktrace); -void TcParser::ReportFastUtf8Error(uint16_t coded_tag, +void TcParser::ReportFastUtf8Error(uint32_t decoded_tag, const TcParseTableBase* table) { - if (coded_tag > 127) { - coded_tag = (coded_tag & 0x7f) + ((coded_tag & 0xff00) >> 1); - } - uint32_t field_num = coded_tag >> 3; + uint32_t field_num = decoded_tag >> 3; const auto* entry = FindFieldEntry(table, field_num); PrintUTF8ErrorLog(MessageName(table), FieldName(table, entry), "parsing", false); @@ -1022,7 +1104,7 @@ PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularString( if (PROTOBUF_PREDICT_TRUE(IsStructurallyValidUTF8(field.Get()))) { return ToParseLoop(PROTOBUF_TC_PARAM_PASS); } - ReportFastUtf8Error(saved_tag, table); + ReportFastUtf8Error(FastDecodeTag(saved_tag), table); return utf8 == kUtf8 ? Error(PROTOBUF_TC_PARAM_PASS) : ToParseLoop(PROTOBUF_TC_PARAM_PASS); } @@ -1053,6 +1135,27 @@ const char* TcParser::FastUS2(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_TC_PARAM_PASS); } +// Inlined string variants: + +const char* TcParser::FastBiS1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); +} +const char* TcParser::FastBiS2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); +} +const char* TcParser::FastSiS1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); +} +const char* TcParser::FastSiS2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); +} +const char* TcParser::FastUiS1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); +} +const char* TcParser::FastUiS2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); +} + template PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedString( PROTOBUF_TC_PARAM_DECL) { @@ -1078,7 +1181,7 @@ PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedString( if (PROTOBUF_PREDICT_TRUE(IsStructurallyValidUTF8(*str))) { break; } - ReportFastUtf8Error(expected_tag, table); + ReportFastUtf8Error(FastDecodeTag(expected_tag), table); if (utf8 == kUtf8) return Error(PROTOBUF_TC_PARAM_PASS); break; } @@ -1615,15 +1718,30 @@ const char* TcParser::MpMessage(PROTOBUF_TC_PARAM_DECL) { if (card == field_layout::kFcRepeated) { PROTOBUF_MUSTTAIL return MpRepeatedMessage(PROTOBUF_TC_PARAM_PASS); } - // Check for wire type mismatch: - // TODO(b/210762816): support groups. - if ((data.tag() & 7) != WireFormatLite::WIRETYPE_LENGTH_DELIMITED) { - PROTOBUF_MUSTTAIL return table->fallback(PROTOBUF_TC_PARAM_PASS); - } - // Lazy and implicit weak fields are handled by generated code: - // TODO(b/210762816): support these. - if ((type_card & field_layout::kRepMask) != field_layout::kRepMessage) { - PROTOBUF_MUSTTAIL return table->fallback(PROTOBUF_TC_PARAM_PASS); + + const uint32_t decoded_tag = data.tag(); + const uint32_t decoded_wiretype = decoded_tag & 7; + const uint16_t rep = type_card & field_layout::kRepMask; + const bool is_group = rep == field_layout::kRepGroup; + + // Validate wiretype: + switch (rep) { + case field_layout::kRepMessage: + if (decoded_wiretype != WireFormatLite::WIRETYPE_LENGTH_DELIMITED) { + goto fallback; + } + break; + case field_layout::kRepGroup: + if (decoded_wiretype != WireFormatLite::WIRETYPE_START_GROUP) { + goto fallback; + } + break; + default: { + fallback: + // Lazy and implicit weak fields are handled by generated code: + // TODO(b/210762816): support these. + PROTOBUF_MUSTTAIL return table->fallback(PROTOBUF_TC_PARAM_PASS); + } } const bool is_oneof = card == field_layout::kFcOneof; @@ -1640,6 +1758,9 @@ const char* TcParser::MpMessage(PROTOBUF_TC_PARAM_DECL) { field = default_instance->New(ctx->data().arena); } SyncHasbits(msg, hasbits, table); + if (is_group) { + return ctx->ParseGroup(field, ptr, decoded_tag); + } return ctx->ParseMessage(field, ptr); } @@ -1648,16 +1769,29 @@ const char* TcParser::MpRepeatedMessage(PROTOBUF_TC_PARAM_DECL) { const uint16_t type_card = entry.type_card; GOOGLE_DCHECK_EQ(type_card & field_layout::kFcMask, static_cast(field_layout::kFcRepeated)); + const uint32_t decoded_tag = data.tag(); + const uint32_t decoded_wiretype = decoded_tag & 7; + const uint16_t rep = type_card & field_layout::kRepMask; + const bool is_group = rep == field_layout::kRepGroup; - // Check for wire type mismatch: - // TODO(b/210762816): support groups. - if ((data.tag() & 7) != WireFormatLite::WIRETYPE_LENGTH_DELIMITED) { - PROTOBUF_MUSTTAIL return table->fallback(PROTOBUF_TC_PARAM_PASS); - } - // Implicit weak fields are handled by generated code: - // TODO(b/210762816): support these. - if ((type_card & field_layout::kRepMask) != field_layout::kRepMessage) { - PROTOBUF_MUSTTAIL return table->fallback(PROTOBUF_TC_PARAM_PASS); + // Validate wiretype: + switch (rep) { + case field_layout::kRepMessage: + if (decoded_wiretype != WireFormatLite::WIRETYPE_LENGTH_DELIMITED) { + goto fallback; + } + break; + case field_layout::kRepGroup: + if (decoded_wiretype != WireFormatLite::WIRETYPE_START_GROUP) { + goto fallback; + } + break; + default: { + fallback: + // Lazy and implicit weak fields are handled by generated code: + // TODO(b/210762816): support these. + PROTOBUF_MUSTTAIL return table->fallback(PROTOBUF_TC_PARAM_PASS); + } } SyncHasbits(msg, hasbits, table); @@ -1666,6 +1800,9 @@ const char* TcParser::MpRepeatedMessage(PROTOBUF_TC_PARAM_DECL) { auto& field = RefAt(msg, entry.offset); MessageLite* value = field.Add>(default_instance); + if (is_group) { + return ctx->ParseGroup(value, ptr, decoded_tag); + } return ctx->ParseMessage(value, ptr); } diff --git a/src/google/protobuf/generated_message_tctable_lite_test.cc b/src/google/protobuf/generated_message_tctable_lite_test.cc index 706248a6b7..4310803a63 100644 --- a/src/google/protobuf/generated_message_tctable_lite_test.cc +++ b/src/google/protobuf/generated_message_tctable_lite_test.cc @@ -44,7 +44,7 @@ namespace { using ::testing::Eq; using ::testing::Not; -MATCHER_P2(IsEntryForFieldNum, table, field_num, +MATCHER_P3(IsEntryForFieldNum, table, field_num, field_numbers_table, StrCat(negation ? "isn't " : "", "the field entry for field number ", field_num)) { if (arg == nullptr) { @@ -54,7 +54,7 @@ MATCHER_P2(IsEntryForFieldNum, table, field_num, // Use the entry's index to compare field numbers. size_t index = static_cast(arg) - &table->field_entries[0]; - uint32_t actual_field_num = table->field_numbers[index]; + uint32_t actual_field_num = field_numbers_table[index]; if (actual_field_num != field_num) { *result_listener << "which is the entry for " << actual_field_num; return false; @@ -64,25 +64,31 @@ MATCHER_P2(IsEntryForFieldNum, table, field_num, TEST(IsEntryForFieldNumTest, Matcher) { // clang-format off - TcParseTable<0, 3, 0, 0> table = { + TcParseTable<0, 3, 0, 0, 2> table = { // header: { 0, 0, 0, 0, // has_bits_offset, extensions 0, // max_field_number - 0, 0, // fast_idx_mask, num_sequential_fields - 0, 0, // sequential_fields_start, num_field_entries + 0, // fast_idx_mask, + offsetof(decltype(table), field_lookup_table), + 0xFFFFFFFF - 7, // 7 = fields 1, 2, and 3. + offsetof(decltype(table), field_names), + 0, // num_field_entries 0, 0, // num_aux_entries, aux_offset, nullptr, // default instance nullptr, // fallback function }}; // clang-format on - table.field_numbers = {1, 2, 3}; + int table_field_numbers[] = {1, 2, 3}; + table.field_lookup_table = {65535, 65535}; - EXPECT_THAT(&table.field_entries[0], IsEntryForFieldNum(&table, 1)); - EXPECT_THAT(&table.field_entries[2], IsEntryForFieldNum(&table, 3)); - EXPECT_THAT(&table.field_entries[1], Not(IsEntryForFieldNum(&table, 3))); + auto& entries = table.field_entries; + EXPECT_THAT(&entries[0], IsEntryForFieldNum(&table, 1, table_field_numbers)); + EXPECT_THAT(&entries[2], IsEntryForFieldNum(&table, 3, table_field_numbers)); + EXPECT_THAT(&entries[1], + Not(IsEntryForFieldNum(&table, 3, table_field_numbers))); - EXPECT_THAT(nullptr, Not(IsEntryForFieldNum(&table, 1))); + EXPECT_THAT(nullptr, Not(IsEntryForFieldNum(&table, 1, table_field_numbers))); } } // namespace @@ -91,30 +97,30 @@ class FindFieldEntryTest : public ::testing::Test { public: // Calls the private `FindFieldEntry` function. template + size_t kNameTableSize, size_t kFieldLookupTableSize> static const TcParseTableBase::FieldEntry* FindFieldEntry( const TcParseTable& table, + kNameTableSize, kFieldLookupTableSize>& table, uint32_t tag) { return TcParser::FindFieldEntry(&table.header, tag); } // Calls the private `FieldName` function. template + size_t kNameTableSize, size_t kFieldLookupTableSize> static StringPiece FieldName( const TcParseTable& table, + kNameTableSize, kFieldLookupTableSize>& table, const TcParseTableBase::FieldEntry* entry) { return TcParser::FieldName(&table.header, entry); } // Calls the private `MessageName` function. template + size_t kNameTableSize, size_t kFieldLookupTableSize> static StringPiece MessageName( const TcParseTable& table) { + kNameTableSize, kFieldLookupTableSize>& table) { return TcParser::MessageName(&table.header); } @@ -123,27 +129,38 @@ class FindFieldEntryTest : public ::testing::Test { }; TEST_F(FindFieldEntryTest, SequentialFieldRange) { - // Look up fields that are within the range of `num_sequential_fields`. + // Look up fields that are within the range of `lookup_table_offset`. // clang-format off - TcParseTable<0, 5, 0, 0> table = { + TcParseTable<0, 5, 0, 0, 8> table = { // header: { 0, 0, 0, 0, // has_bits_offset, extensions 111, // max_field_number - 0, 4, // fast_idx_mask, num_sequential_fields - 2, 5, // sequential_fields_start, num_field_entries + 0, // fast_idx_mask, + offsetof(decltype(table), field_lookup_table), + 0xFFFFFFFF - (1 << 1) - (1 << 2) // fields 2, 3 + - (1 << 3) - (1 << 4), // fields 4, 5 + offsetof(decltype(table), field_entries), + 5, // num_field_entries 0, 0, // num_aux_entries, aux_offset, nullptr, // default instance {}, // fallback function }, {}, // fast_entries - // field_numbers: - {{2, 3, 4, 5, 111}}, + // field_lookup_table for 2, 3, 4, 5, 111: + {{ + 111, 0, // field 111 + 1, // 1 skip entry + 0xFFFE, 4, // 1 field, entry 4. + 65535, 65535, // end of table + }}, }; // clang-format on + int table_field_numbers[] = {2, 3, 4, 5, 111}; - for (int i : table.field_numbers) { - EXPECT_THAT(FindFieldEntry(table, i), IsEntryForFieldNum(&table, i)); + for (int i : table_field_numbers) { + EXPECT_THAT(FindFieldEntry(table, i), + IsEntryForFieldNum(&table, i, table_field_numbers)); } for (int i : {0, 1, 6, 7, 110, 112, 500000000}) { GOOGLE_LOG(WARNING) << "Field " << i; @@ -152,33 +169,43 @@ TEST_F(FindFieldEntryTest, SequentialFieldRange) { } TEST_F(FindFieldEntryTest, SmallScanRange) { - // Look up fields past `num_sequential_fields`, but before binary search. + // Look up fields past `lookup_table_offset`, but before binary search. ASSERT_THAT(small_scan_size(), Eq(4)) << "test needs to be updated"; // clang-format off - TcParseTable<0, 6, 0, 0> table = { + TcParseTable<0, 6, 0, 0, 8> table = { // header: { 0, 0, 0, 0, // has_bits_offset, extensions 111, // max_field_number - 0, 1, // fast_idx_mask, num_sequential_fields - 1, 6, // sequential_fields_start, num_field_entries + 0, // fast_idx_mask, + offsetof(decltype(table), field_lookup_table), + 0xFFFFFFFF - (1<<0) - (1<<2) - (1<<3) - (1<<4) - (1<<6), // 1,3-5,7 + offsetof(decltype(table), field_entries), + 6, // num_field_entries 0, 0, // num_aux_entries, aux_offset, nullptr, // default instance {}, // fallback function }, {}, // fast_entries - // field_numbers: - {{// Sequential entries: - 1, - // Small scan range: - 3, 4, 5, 7, - // Binary search range: - 111}}, + // field_lookup_table for 1, 3, 4, 5, 7, 111: + {{ + 111, 0, // field 111 + 1, // 1 skip entry + 0xFFFE, 5, // 1 field, entry 5 + 65535, 65535 // end of table + }}, }; // clang-format on + int table_field_numbers[] = {// Sequential entries: + 1, + // Small scan range: + 3, 4, 5, 7, + // Binary search range: + 111}; - for (int i : table.field_numbers) { - EXPECT_THAT(FindFieldEntry(table, i), IsEntryForFieldNum(&table, i)); + for (int i : table_field_numbers) { + EXPECT_THAT(FindFieldEntry(table, i), + IsEntryForFieldNum(&table, i, table_field_numbers)); } for (int i : {0, 2, 6, 8, 9, 110, 112, 500000000}) { EXPECT_THAT(FindFieldEntry(table, i), Eq(nullptr)); @@ -191,29 +218,43 @@ TEST_F(FindFieldEntryTest, BinarySearchRange) { ASSERT_THAT(small_scan_size(), Eq(4)) << "test needs to be updated"; // clang-format off - TcParseTable<0, 10, 0, 0> table = { + TcParseTable<0, 10, 0, 0, 8> table = { // header: { 0, 0, 0, 0, // has_bits_offset, extensions 70, // max_field_number - 0, 1, // fast_idx_mask, num_sequential_fields - 1, 10, // sequential_fields_start, num_field_entries + 0, // fast_idx_mask, + offsetof(decltype(table), field_lookup_table), + 0xFFFFFFFF - (1<<0) - (1<<2) - (1<<3) - (1<<4) // 1, 3, 4, 5, 6 + - (1<<5) - (1<<7) - (1<<8) - (1<<10) // 8, 9, 11, 12 + - (1<<11), + offsetof(decltype(table), field_entries), + 10, // num_field_entries 0, 0, // num_aux_entries, aux_offset, nullptr, // default instance {}, // fallback function }, {}, // fast_entries - // field_numbers: - {{// Sequential entries: + // field_lookup_table for 1, 3, 4, 5, 6, 8, 9, 11, 12, 70 + {{ + 70, 0, // field 70 + 1, // 1 skip entry + 0xFFFE, 9, // 1 field, entry 9 + 65535, 65535 // end of table + }}, + }; + int table_field_numbers[] = { + // Sequential entries: 1, // Small scan range: 3, 4, 5, 6, // Binary search range: - 8, 9, 11, 12, 70}}, + 8, 9, 11, 12, 70 }; // clang-format on - for (int i : table.field_numbers) { - EXPECT_THAT(FindFieldEntry(table, i), IsEntryForFieldNum(&table, i)); + for (int i : table_field_numbers) { + EXPECT_THAT(FindFieldEntry(table, i), + IsEntryForFieldNum(&table, i, table_field_numbers)); } for (int i : {0, 2, 7, 10, 13, 69, 71, 112, 500000000}) { EXPECT_THAT(FindFieldEntry(table, i), Eq(nullptr)); @@ -223,21 +264,25 @@ TEST_F(FindFieldEntryTest, BinarySearchRange) { TEST_F(FindFieldEntryTest, OutOfRange) { // Look up tags that are larger than the maximum in the message. // clang-format off - TcParseTable<0, 3, 0, 15> table = { + TcParseTable<0, 3, 0, 15, 2> table = { // header: { 0, 0, 0, 0, // has_bits_offset, extensions 3, // max_field_number - 0, 3, // fast_idx_mask, num_sequential_fields - 1, 3, // sequential_fields_start, num_field_entries + 0, // fast_idx_mask, + offsetof(decltype(table), field_lookup_table), + 0xFFFFFFFF - (1<<0) - (1<<1) - (1<<2), // fields 1, 2, 3 + offsetof(decltype(table), field_entries), + 3, // num_field_entries 0, // num_aux_entries offsetof(decltype(table), field_names), // no aux_entries nullptr, // default instance {}, // fallback function }, {}, // fast_entries - // field_numbers: - {{1, 2, 3}}, + {{// field lookup table + 65535, 65535 // end of table + }}, {}, // "mini" table // auxiliary entries (none in this test) {{ // name lengths @@ -248,10 +293,12 @@ TEST_F(FindFieldEntryTest, OutOfRange) { "003"}}, }; // clang-format on + int table_field_numbers[] = {1, 2, 3}; - for (int field_num : table.field_numbers) { + for (int field_num : table_field_numbers) { auto* entry = FindFieldEntry(table, field_num); - EXPECT_THAT(entry, IsEntryForFieldNum(&table, field_num)); + EXPECT_THAT(entry, + IsEntryForFieldNum(&table, field_num, table_field_numbers)); StringPiece name = FieldName(table, entry); EXPECT_EQ(name.length(), field_num); @@ -265,21 +312,27 @@ TEST_F(FindFieldEntryTest, OutOfRange) { TEST_F(FindFieldEntryTest, EmptyMessage) { // Ensure that tables with no fields are handled correctly. - using TableType = TcParseTable<0, 0, 0, 20>; + using TableType = TcParseTable<0, 0, 0, 20, 2>; // clang-format off TableType table = { // header: { 0, 0, 0, 0, // has_bits_offset, extensions 0, // max_field_number - 0, 0, // fast_idx_mask, num_sequential_fields - 0, 0, // sequential_fields_start, num_field_entries + 0, // fast_idx_mask, + offsetof(decltype(table), field_lookup_table), + 0xFFFFFFFF, // no fields + offsetof(decltype(table), field_names), // no field_entries + 0, // num_field_entries 0, // num_aux_entries offsetof(TableType, field_names), nullptr, // default instance nullptr, // fallback function }, {}, // fast_entries + {{// empty field lookup table + 65535, 65535 + }}, {{ "\13\0\0\0\0\0\0\0" "MessageName" @@ -294,13 +347,32 @@ TEST_F(FindFieldEntryTest, EmptyMessage) { } // Make a monster with lots of field numbers + +int32_t test_all_types_table_field_numbers[] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, // + 11, 12, 13, 14, 15, 18, 19, 21, 22, 24, // + 25, 27, 31, 32, 33, 34, 35, 36, 37, 38, // + 39, 40, 41, 42, 43, 44, 45, 48, 49, 51, // + 52, 54, 55, 56, 57, 58, 59, 60, 61, 62, // + 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, // + 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, // + 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, // + 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, // + 111, 112, 113, 114, 115, 116, 117, 118, 119, 201, // + 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, // + 251, 252, 253, 254, 255, 321, 322, 401, 402, 403, // + 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, // + 414, 415, 416, 417}; + // clang-format off -const TcParseTable<5, 134, 5, 2176> test_all_types_table = { +const TcParseTable<5, 134, 5, 2176, 55> test_all_types_table = { // header: { 0, 0, 0, 0, // has_bits_offset, extensions 418, 248, // max_field_number, fast_idx_mask - 14, 1, // num_sequential_fields, sequential_fields_start + offsetof(decltype(test_all_types_table), field_lookup_table), + 977895424, // skipmap for fields 1-15,18-19,21-22,24-25,27,31-32 + offsetof(decltype(test_all_types_table), field_entries), 135, // num_field_entries 5, // num_aux_entries offsetof(decltype(test_all_types_table), aux_entries), @@ -310,21 +382,18 @@ const TcParseTable<5, 134, 5, 2176> test_all_types_table = { {{ // tail-call table }}, - {{// field numbers - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 18, 19, 21, 22, 24, - 25, 27, 31, 32, 33, 34, 35, 36, 37, 38, - 39, 40, 41, 42, 43, 44, 45, 48, 49, 51, - 52, 54, 55, 56, 57, 58, 59, 60, 61, 62, - 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, - 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, - 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, - 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, - 111, 112, 113, 114, 115, 116, 117, 118, 119, 201, - 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, - 251, 252, 253, 254, 255, 321, 322, 401, 402, 403, - 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, - 414, 415, 416, 417}}, + {{ // field lookup table + // + // fields 33-417, over 25 skipmap / offset pairs + 33, 0, 25, + 24576, 24, 18, 38, 0, 52, 0, 68, 16320, 84, + 65408, 92, 65535, 99, 65535, 99, 65535, 99, 65535, 99, + 65279, 99, 65535, 100, 65535, 100, 32768, 100, 65535, 115, + 65535, 115, 65535, 115, 65535, 115, 65532, 115, 65535, 117, + 65535, 117, 65535, 117, 65535, 117, 0, 117, 65532, 133, + // end of table + 65535, 65535 + }}, {{ // "mini" table }}, diff --git a/src/google/protobuf/inlined_string_field_unittest.cc b/src/google/protobuf/inlined_string_field_unittest.cc index 52f16f3993..9efe6e9324 100644 --- a/src/google/protobuf/inlined_string_field_unittest.cc +++ b/src/google/protobuf/inlined_string_field_unittest.cc @@ -53,8 +53,6 @@ namespace protobuf { using internal::ArenaStringPtr; using internal::InlinedStringField; -static std::string WrapString(const char* value) { return value; } - namespace { } // namespace } // namespace protobuf diff --git a/src/google/protobuf/io/io_win32_unittest.cc b/src/google/protobuf/io/io_win32_unittest.cc index c50f68d6a9..e8f378fcac 100644 --- a/src/google/protobuf/io/io_win32_unittest.cc +++ b/src/google/protobuf/io/io_win32_unittest.cc @@ -164,8 +164,8 @@ void IoWin32Test::SetUp() { test_tmpdir.clear(); wtest_tmpdir.clear(); DWORD size = ::GetCurrentDirectoryW(MAX_PATH, working_directory); - EXPECT_GT(size, 0); - EXPECT_LT(size, MAX_PATH); + EXPECT_GT(size, 0U); + EXPECT_LT(size, static_cast(MAX_PATH)); string tmp; bool ok = false; @@ -581,7 +581,7 @@ TEST_F(IoWin32Test, ExpandWildcardsFailsIfNoFileMatchesTest) { TEST_F(IoWin32Test, AsWindowsPathTest) { DWORD size = GetCurrentDirectoryW(0, nullptr); std::unique_ptr cwd_str(new wchar_t[size]); - EXPECT_GT(GetCurrentDirectoryW(size, cwd_str.get()), 0); + EXPECT_GT(GetCurrentDirectoryW(size, cwd_str.get()), 0U); wstring cwd = wstring(L"\\\\?\\") + cwd_str.get(); ASSERT_EQ(testonly_utf8_to_winpath("relative_mkdirtest"), diff --git a/src/google/protobuf/io/tokenizer.cc b/src/google/protobuf/io/tokenizer.cc index d6456bcc94..6b4b92e65c 100644 --- a/src/google/protobuf/io/tokenizer.cc +++ b/src/google/protobuf/io/tokenizer.cc @@ -92,8 +92,8 @@ #include #include -#include #include +#include #include #include #include @@ -686,7 +686,7 @@ bool Tokenizer::Next() { error_collector_->AddError( line_, column_, StringPrintf("Interpreting non ascii codepoint %d.", - static_cast(current_char_))); + static_cast(current_char_))); } NextChar(); current_.type = TYPE_SYMBOL; diff --git a/src/google/protobuf/io/zero_copy_stream_unittest.cc b/src/google/protobuf/io/zero_copy_stream_unittest.cc index fe9b6a191e..d4e5b54514 100644 --- a/src/google/protobuf/io/zero_copy_stream_unittest.cc +++ b/src/google/protobuf/io/zero_copy_stream_unittest.cc @@ -49,7 +49,7 @@ #include #include -#ifndef _MSC_VER +#ifndef _WIN32 #include #include #endif @@ -764,7 +764,7 @@ TEST_F(IoTest, FileIo) { } } -#ifndef _MSC_VER +#ifndef _WIN32 // This tests the FileInputStream with a non blocking file. It opens a pipe in // non blocking mode, then starts reading it. The writing thread starts writing // 100ms after that. diff --git a/src/google/protobuf/parse_context.h b/src/google/protobuf/parse_context.h index f79b8658e8..c00048d167 100644 --- a/src/google/protobuf/parse_context.h +++ b/src/google/protobuf/parse_context.h @@ -804,7 +804,7 @@ inline bool VerifyUTF8(const std::string* s, const char* field_name) { } // All the string parsers with or without UTF checking and for all CTypes. -PROTOBUF_EXPORT PROTOBUF_NODISCARD const char* InlineGreedyStringParser( +PROTOBUF_NODISCARD PROTOBUF_EXPORT const char* InlineGreedyStringParser( std::string* s, const char* ptr, ParseContext* ctx); @@ -876,19 +876,19 @@ PROTOBUF_NODISCARD const char* WireFormatParser(T& field_parser, // corresponding field // These are packed varints -PROTOBUF_EXPORT PROTOBUF_NODISCARD const char* PackedInt32Parser( +PROTOBUF_NODISCARD PROTOBUF_EXPORT const char* PackedInt32Parser( void* object, const char* ptr, ParseContext* ctx); -PROTOBUF_EXPORT PROTOBUF_NODISCARD const char* PackedUInt32Parser( +PROTOBUF_NODISCARD PROTOBUF_EXPORT const char* PackedUInt32Parser( void* object, const char* ptr, ParseContext* ctx); -PROTOBUF_EXPORT PROTOBUF_NODISCARD const char* PackedInt64Parser( +PROTOBUF_NODISCARD PROTOBUF_EXPORT const char* PackedInt64Parser( void* object, const char* ptr, ParseContext* ctx); -PROTOBUF_EXPORT PROTOBUF_NODISCARD const char* PackedUInt64Parser( +PROTOBUF_NODISCARD PROTOBUF_EXPORT const char* PackedUInt64Parser( void* object, const char* ptr, ParseContext* ctx); -PROTOBUF_EXPORT PROTOBUF_NODISCARD const char* PackedSInt32Parser( +PROTOBUF_NODISCARD PROTOBUF_EXPORT const char* PackedSInt32Parser( void* object, const char* ptr, ParseContext* ctx); -PROTOBUF_EXPORT PROTOBUF_NODISCARD const char* PackedSInt64Parser( +PROTOBUF_NODISCARD PROTOBUF_EXPORT const char* PackedSInt64Parser( void* object, const char* ptr, ParseContext* ctx); -PROTOBUF_EXPORT PROTOBUF_NODISCARD const char* PackedEnumParser( +PROTOBUF_NODISCARD PROTOBUF_EXPORT const char* PackedEnumParser( void* object, const char* ptr, ParseContext* ctx); template @@ -922,28 +922,28 @@ PROTOBUF_NODISCARD const char* PackedEnumParserArg( }); } -PROTOBUF_EXPORT PROTOBUF_NODISCARD const char* PackedBoolParser( +PROTOBUF_NODISCARD PROTOBUF_EXPORT const char* PackedBoolParser( void* object, const char* ptr, ParseContext* ctx); -PROTOBUF_EXPORT PROTOBUF_NODISCARD const char* PackedFixed32Parser( +PROTOBUF_NODISCARD PROTOBUF_EXPORT const char* PackedFixed32Parser( void* object, const char* ptr, ParseContext* ctx); -PROTOBUF_EXPORT PROTOBUF_NODISCARD const char* PackedSFixed32Parser( +PROTOBUF_NODISCARD PROTOBUF_EXPORT const char* PackedSFixed32Parser( void* object, const char* ptr, ParseContext* ctx); -PROTOBUF_EXPORT PROTOBUF_NODISCARD const char* PackedFixed64Parser( +PROTOBUF_NODISCARD PROTOBUF_EXPORT const char* PackedFixed64Parser( void* object, const char* ptr, ParseContext* ctx); -PROTOBUF_EXPORT PROTOBUF_NODISCARD const char* PackedSFixed64Parser( +PROTOBUF_NODISCARD PROTOBUF_EXPORT const char* PackedSFixed64Parser( void* object, const char* ptr, ParseContext* ctx); -PROTOBUF_EXPORT PROTOBUF_NODISCARD const char* PackedFloatParser( +PROTOBUF_NODISCARD PROTOBUF_EXPORT const char* PackedFloatParser( void* object, const char* ptr, ParseContext* ctx); -PROTOBUF_EXPORT PROTOBUF_NODISCARD const char* PackedDoubleParser( +PROTOBUF_NODISCARD PROTOBUF_EXPORT const char* PackedDoubleParser( void* object, const char* ptr, ParseContext* ctx); // This is the only recursive parser. -PROTOBUF_EXPORT PROTOBUF_NODISCARD const char* UnknownGroupLiteParse( +PROTOBUF_NODISCARD PROTOBUF_EXPORT const char* UnknownGroupLiteParse( std::string* unknown, const char* ptr, ParseContext* ctx); // This is a helper to for the UnknownGroupLiteParse but is actually also // useful in the generated code. It uses overload on std::string* vs // UnknownFieldSet* to make the generated code isomorphic between full and lite. -PROTOBUF_EXPORT PROTOBUF_NODISCARD const char* UnknownFieldParse( +PROTOBUF_NODISCARD PROTOBUF_EXPORT const char* UnknownFieldParse( uint32_t tag, std::string* unknown, const char* ptr, ParseContext* ctx); } // namespace internal diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index e756603224..430a3e8d97 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -157,17 +157,17 @@ #ifdef PROTOBUF_VERSION #error PROTOBUF_VERSION was previously defined #endif -#define PROTOBUF_VERSION 3018001 +#define PROTOBUF_VERSION 3019004 #ifdef PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC #error PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC was previously defined #endif -#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC 3018000 +#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC 3019000 #ifdef PROTOBUF_MIN_PROTOC_VERSION #error PROTOBUF_MIN_PROTOC_VERSION was previously defined #endif -#define PROTOBUF_MIN_PROTOC_VERSION 3018000 +#define PROTOBUF_MIN_PROTOC_VERSION 3019000 #ifdef PROTOBUF_VERSION_SUFFIX #error PROTOBUF_VERSION_SUFFIX was previously defined @@ -233,11 +233,14 @@ #ifdef PROTOBUF_TAILCALL #error PROTOBUF_TAILCALL was previously defined #endif -#if __has_cpp_attribute(clang::musttail) && \ - !defined(__arm__) && !defined(_ARCH_PPC) && !defined(__wasm__) +#if __has_cpp_attribute(clang::musttail) && !defined(__arm__) && \ + !defined(_ARCH_PPC) && !defined(__wasm__) && \ + !(defined(_MSC_VER) && defined(_M_IX86)) # ifndef PROTO2_OPENSOURCE // Compilation fails on ARM32: b/195943306 // Compilation fails on powerpc64le: b/187985113 +// Compilation fails on X86 Windows: +// https://github.com/llvm/llvm-project/issues/53271 # endif #define PROTOBUF_MUSTTAIL [[clang::musttail]] #define PROTOBUF_TAILCALL true @@ -360,7 +363,7 @@ // The minimum library version which works with the current version of the // headers. -#define GOOGLE_PROTOBUF_MIN_LIBRARY_VERSION 3018000 +#define GOOGLE_PROTOBUF_MIN_LIBRARY_VERSION 3019000 #ifdef PROTOBUF_RTTI #error PROTOBUF_RTTI was previously defined @@ -663,7 +666,7 @@ #ifdef PROTOBUF_PRAGMA_INIT_SEG #error PROTOBUF_PRAGMA_INIT_SEG was previously defined #endif -#if _MSC_VER +#ifdef _MSC_VER #define PROTOBUF_PRAGMA_INIT_SEG __pragma(init_seg(lib)) #else #define PROTOBUF_PRAGMA_INIT_SEG @@ -832,7 +835,7 @@ #endif // Silence some MSVC warnings in all our code. -#if _MSC_VER +#ifdef _MSC_VER #pragma warning(push) // For non-trivial unions #pragma warning(disable : 4582) diff --git a/src/google/protobuf/port_undef.inc b/src/google/protobuf/port_undef.inc index 6b05be686b..664c9364f4 100644 --- a/src/google/protobuf/port_undef.inc +++ b/src/google/protobuf/port_undef.inc @@ -146,6 +146,6 @@ #endif // Pop the warning(push) from port_def.inc -#if _MSC_VER +#ifdef _MSC_VER #pragma warning(pop) #endif diff --git a/src/google/protobuf/source_context.pb.h b/src/google/protobuf/source_context.pb.h index 5bdc1ed9a9..1020bf0453 100644 --- a/src/google/protobuf/source_context.pb.h +++ b/src/google/protobuf/source_context.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3018000 +#if PROTOBUF_VERSION < 3019000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3018001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3019004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/struct.pb.h b/src/google/protobuf/struct.pb.h index 89c835b1ab..e5b0a38153 100644 --- a/src/google/protobuf/struct.pb.h +++ b/src/google/protobuf/struct.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3018000 +#if PROTOBUF_VERSION < 3019000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3018001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3019004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/stubs/common.h b/src/google/protobuf/stubs/common.h index 1d25852264..6fa71ee873 100644 --- a/src/google/protobuf/stubs/common.h +++ b/src/google/protobuf/stubs/common.h @@ -82,7 +82,7 @@ namespace internal { // The current version, represented as a single integer to make comparison // easier: major * 10^6 + minor * 10^3 + micro -#define GOOGLE_PROTOBUF_VERSION 3018001 +#define GOOGLE_PROTOBUF_VERSION 3019004 // A suffix string for alpha, beta or rc releases. Empty for stable releases. #define GOOGLE_PROTOBUF_VERSION_SUFFIX "" @@ -90,15 +90,15 @@ namespace internal { // The minimum header version which works with the current version of // the library. This constant should only be used by protoc's C++ code // generator. -static const int kMinHeaderVersionForLibrary = 3018000; +static const int kMinHeaderVersionForLibrary = 3019000; // The minimum protoc version which works with the current version of the // headers. -#define GOOGLE_PROTOBUF_MIN_PROTOC_VERSION 3018000 +#define GOOGLE_PROTOBUF_MIN_PROTOC_VERSION 3019000 // The minimum header version which works with the current version of // protoc. This constant should only be used in VerifyVersion(). -static const int kMinHeaderVersionForProtoc = 3018000; +static const int kMinHeaderVersionForProtoc = 3019000; // Verifies that the headers and libraries are compatible. Use the macro // below to call this. @@ -189,10 +189,6 @@ class FatalException : public std::exception { }; #endif -// This is at the end of the file instead of the beginning to work around a bug -// in some versions of MSVC. -using std::string; - } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/stubs/int128.cc b/src/google/protobuf/stubs/int128.cc index e5e159e1f8..b60722defb 100644 --- a/src/google/protobuf/stubs/int128.cc +++ b/src/google/protobuf/stubs/int128.cc @@ -173,12 +173,13 @@ std::ostream& operator<<(std::ostream& o, const uint128& b) { // Add the requisite padding. std::streamsize width = o.width(0); - if (width > rep.size()) { + auto repSize = static_cast(rep.size()); + if (width > repSize) { if ((flags & std::ios::adjustfield) == std::ios::left) { - rep.append(width - rep.size(), o.fill()); + rep.append(width - repSize, o.fill()); } else { - rep.insert(static_cast(0), - width - rep.size(), o.fill()); + rep.insert(static_cast(0), width - repSize, + o.fill()); } } diff --git a/src/google/protobuf/stubs/stl_util.h b/src/google/protobuf/stubs/stl_util.h index 3bc1dbf002..e6260d0760 100644 --- a/src/google/protobuf/stubs/stl_util.h +++ b/src/google/protobuf/stubs/stl_util.h @@ -37,6 +37,9 @@ #include +// Must be last. +#include // NOLINT + namespace google { namespace protobuf { @@ -57,7 +60,7 @@ inline void STLStringResizeUninitializedAmortized(std::string* s, const size_t cap = s->capacity(); if (new_size > cap) { // Make sure to always grow by at least a factor of 2x. - s->reserve(std::max(new_size, 2 * cap)); + s->reserve(std::max(new_size, 2 * cap)); } STLStringResizeUninitialized(s, new_size); } @@ -82,4 +85,6 @@ inline char* string_as_array(std::string* str) { } // namespace protobuf } // namespace google +#include // NOLINT + #endif // GOOGLE_PROTOBUF_STUBS_STL_UTIL_H__ diff --git a/src/google/protobuf/timestamp.pb.h b/src/google/protobuf/timestamp.pb.h index f8a5812d48..41545fc407 100644 --- a/src/google/protobuf/timestamp.pb.h +++ b/src/google/protobuf/timestamp.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3018000 +#if PROTOBUF_VERSION < 3019000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3018001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3019004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/type.pb.h b/src/google/protobuf/type.pb.h index d1d4d502a1..e288097673 100644 --- a/src/google/protobuf/type.pb.h +++ b/src/google/protobuf/type.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3018000 +#if PROTOBUF_VERSION < 3019000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3018001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3019004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/util/internal/protostream_objectsource.cc b/src/google/protobuf/util/internal/protostream_objectsource.cc index 99db5932a7..cf6c99ac1f 100644 --- a/src/google/protobuf/util/internal/protostream_objectsource.cc +++ b/src/google/protobuf/util/internal/protostream_objectsource.cc @@ -36,7 +36,6 @@ #include #include -#include #include #include #include @@ -47,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -1098,11 +1098,11 @@ const std::string FormatNanos(uint32_t nanos, bool with_trailing_zeros) { return with_trailing_zeros ? ".000" : ""; } - const char* format = (nanos % 1000 != 0) ? "%.9f" - : (nanos % 1000000 != 0) ? "%.6f" - : "%.3f"; - std::string formatted = - StringPrintf(format, static_cast(nanos) / kNanosPerSecond); + const int precision = (nanos % 1000 != 0) ? 9 + : (nanos % 1000000 != 0) ? 6 + : 3; + std::string formatted = StringPrintf( + "%.*f", precision, static_cast(nanos) / kNanosPerSecond); // remove the leading 0 before decimal. return formatted.substr(1); } diff --git a/src/google/protobuf/util/message_differencer.cc b/src/google/protobuf/util/message_differencer.cc index 56e927e782..d5a119d375 100644 --- a/src/google/protobuf/util/message_differencer.cc +++ b/src/google/protobuf/util/message_differencer.cc @@ -44,7 +44,6 @@ #include #include -#include #include #include #include @@ -56,6 +55,7 @@ #include #include #include +#include #include // Always include as last one, otherwise it can break compilation diff --git a/src/google/protobuf/util/time_util.cc b/src/google/protobuf/util/time_util.cc index 6b9c6654b6..61ce8eca59 100644 --- a/src/google/protobuf/util/time_util.cc +++ b/src/google/protobuf/util/time_util.cc @@ -32,11 +32,11 @@ #include -#include #include #include #include #include +#include #include // Must go after other includes. diff --git a/src/google/protobuf/wrappers.pb.h b/src/google/protobuf/wrappers.pb.h index 8479fc5c29..8e5f74388f 100644 --- a/src/google/protobuf/wrappers.pb.h +++ b/src/google/protobuf/wrappers.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3018000 +#if PROTOBUF_VERSION < 3019000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3018001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3019004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/toolchain/BUILD b/toolchain/BUILD index 467d1a5df7..ff2b51542a 100644 --- a/toolchain/BUILD +++ b/toolchain/BUILD @@ -44,15 +44,16 @@ cc_toolchain_config( bit_flag = "-m64", cpp_flag = "-lstdc++", extra_compiler_flags = [ - "-I/usr/aarch64-linux-gnu/include/c++/8/aarch64-linux-gnu/", - "-I/usr/aarch64-linux-gnu/include/c++/8" + "-I/opt/manylinux/2014/aarch64/usr/include/c++/10/aarch64-redhat-linux", + "-I/opt/manylinux/2014/aarch64/usr/include/c++/10" ], - extra_include = "/usr/include", + sysroot = "/opt/manylinux/2014/aarch64", linker_path = "/usr/bin/ld", target_cpu = "aarch64", target_full_name = "aarch64-linux-gnu", - toolchain_dir = "/usr/aarch64-linux-gnu/include", toolchain_name = "linux_aarch_64", + # Don't really need this, setting it because it's required. + toolchain_dir = "/opt/manylinux/2014/aarch64/usr/include", ) cc_toolchain_config( diff --git a/toolchain/cc_toolchain_config.bzl b/toolchain/cc_toolchain_config.bzl index 1d54c1d70d..27fe274e6e 100644 --- a/toolchain/cc_toolchain_config.bzl +++ b/toolchain/cc_toolchain_config.bzl @@ -108,7 +108,7 @@ def _impl(ctx): enabled = (ctx.attr.sysroot != ""), flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + all_compile_actions, flag_groups = [ flag_group( flags = [