From 77f30b686b2260215a975d6dfd42a84cd6aa90f7 Mon Sep 17 00:00:00 2001 From: Yannic Date: Thu, 9 Jan 2020 19:23:02 +0100 Subject: [PATCH 01/27] Blacklist .proto source files if Bazel allows us to (#7065) * Blacklist .proto source files is Bazel allows us to This is a partial revert of https://github.com/protocolbuffers/protobuf/commit/7b28278c7d4f4175e70aef2f89d304696eb85ae3 to unblock, e.g., https://github.com/grpc/grpc/pull/21590 or https://github.com/lyft/envoy-mobile/issues/617 until Bazel is fixed. Note: this is a forward-compatible change that automatically switches to the behavior intended by https://github.com/protocolbuffers/protobuf/commit/7b28278c7d4f4175e70aef2f89d304696eb85ae3 when a compatible Bazel is released without requiring users to upgrade Protobuf. We will revert this change when Bazel is fixed. * Remove trailing , * Update BUILD --- BUILD | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/BUILD b/BUILD index 1c5236d9d8..e678db7015 100644 --- a/BUILD +++ b/BUILD @@ -3,6 +3,7 @@ load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test", "objc_library") load("@rules_java//java:defs.bzl", "java_library") load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain", "proto_library") +load("@rules_proto//proto/private:native.bzl", "native_proto_common") load("@rules_python//python:defs.bzl", "py_library") licenses(["notice"]) @@ -986,9 +987,13 @@ cc_library( ], ) +# Note: We use `native_proto_common` here because we depend on an implementation-detail of +# `proto_lang_toolchain`, which may not be available on `proto_common`. +reject_blacklisted_files = not hasattr(native_proto_common, "proto_lang_toolchain_rejects_files_do_not_use_or_we_will_break_you_without_mercy") +cc_toolchain_blacklisted_protos = [proto + "_proto" for proto in WELL_KNOWN_PROTO_MAP.keys()] if reject_blacklisted_files else [":well_known_protos"] proto_lang_toolchain( name = "cc_toolchain", - blacklisted_protos = [proto + "_proto" for proto in WELL_KNOWN_PROTO_MAP.keys()], + blacklisted_protos = cc_toolchain_blacklisted_protos, command_line = "--cpp_out=$(OUT)", runtime = ":protobuf", visibility = ["//visibility:public"], From abc5062a61ba2ac01cb6892683267bbfc4838bb6 Mon Sep 17 00:00:00 2001 From: Jingwen Chen Date: Mon, 2 Dec 2019 12:35:00 -0500 Subject: [PATCH 02/27] Migrate from maven_jar to jvm_maven_import_external to prepare for Bazel 2.0 --- WORKSPACE | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 35ef1d583c..aedc4e0729 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -15,6 +15,7 @@ load("//:protobuf_deps.bzl", "protobuf_deps") # Load common dependencies. protobuf_deps() +load("@bazel_tools//tools/build_defs/repo:jvm.bzl", "jvm_maven_import_external") bind( name = "python_headers", @@ -31,9 +32,14 @@ bind( actual = "@submodule_gmock//:gtest_main", ) -maven_jar( +jvm_maven_import_external( name = "guava_maven", artifact = "com.google.guava:guava:18.0", + artifact_sha256 = "d664fbfc03d2e5ce9cab2a44fb01f1d0bf9dfebeccc1a473b1f9ea31f79f6f99", + server_urls = [ + "https://jcenter.bintray.com/", + "https://repo1.maven.org/maven2", + ], ) bind( @@ -41,9 +47,14 @@ bind( actual = "@guava_maven//jar", ) -maven_jar( +jvm_maven_import_external( name = "gson_maven", artifact = "com.google.code.gson:gson:2.7", + artifact_sha256 = "2d43eb5ea9e133d2ee2405cc14f5ee08951b8361302fdd93494a3a997b508d32", + server_urls = [ + "https://jcenter.bintray.com/", + "https://repo1.maven.org/maven2", + ], ) bind( @@ -51,9 +62,14 @@ bind( actual = "@gson_maven//jar", ) -maven_jar( +jvm_maven_import_external( name = "error_prone_annotations_maven", artifact = "com.google.errorprone:error_prone_annotations:2.3.2", + artifact_sha256 = "357cd6cfb067c969226c442451502aee13800a24e950fdfde77bcdb4565a668d", + server_urls = [ + "https://jcenter.bintray.com/", + "https://repo1.maven.org/maven2", + ], ) bind( From 80eca4c1cd7de569f7e17c524bc731fdcdb97fed Mon Sep 17 00:00:00 2001 From: Scott Hart Date: Mon, 13 Jan 2020 18:21:40 -0500 Subject: [PATCH 03/27] bug: #7076 adds OUT and OPTIONAL to windows portability files (#7088) --- src/google/protobuf/port_def.inc | 4 ++++ src/google/protobuf/port_undef.inc | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 887a372696..3e39f915f6 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -388,6 +388,10 @@ #undef IGNORE #pragma push_macro("IN") #undef IN +#pragma push_macro("OUT") +#undef OUT +#pragma push_macro("OPTIONAL") +#undef OPTIONAL #pragma push_macro("min") #undef min #pragma push_macro("max") diff --git a/src/google/protobuf/port_undef.inc b/src/google/protobuf/port_undef.inc index dc61ba18d4..80822b470a 100644 --- a/src/google/protobuf/port_undef.inc +++ b/src/google/protobuf/port_undef.inc @@ -74,6 +74,8 @@ #pragma pop_macro("GetMessage") #pragma pop_macro("IGNORE") #pragma pop_macro("IN") +#pragma pop_macro("OUT") +#pragma pop_macro("OPTIONAL") #pragma pop_macro("min") #pragma pop_macro("max") #endif From 29cd005ce1fe1a8fabf11e325cb13006a6646d59 Mon Sep 17 00:00:00 2001 From: Yannic Date: Wed, 15 Jan 2020 19:22:16 +0100 Subject: [PATCH 04/27] [bazel] Fix blacklisted_protos in cc_toolchain and add test (#7096) --- BUILD | 16 ++++++++++++++-- WORKSPACE | 5 +++++ cc_proto_blacklist_test.bzl | 38 +++++++++++++++++++++++++++++++++++++ kokoro/linux/bazel/build.sh | 5 ++++- protobuf_deps.bzl | 8 +++++--- 5 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 cc_proto_blacklist_test.bzl diff --git a/BUILD b/BUILD index e678db7015..79871d621c 100644 --- a/BUILD +++ b/BUILD @@ -1,10 +1,11 @@ # Bazel (https://bazel.build/) BUILD file for Protobuf. -load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test", "objc_library") +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test", "objc_library", native_cc_proto_library = "cc_proto_library") load("@rules_java//java:defs.bzl", "java_library") load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain", "proto_library") load("@rules_proto//proto/private:native.bzl", "native_proto_common") load("@rules_python//python:defs.bzl", "py_library") +load(":cc_proto_blacklist_test.bzl", "cc_proto_blacklist_test") licenses(["notice"]) @@ -315,6 +316,17 @@ cc_proto_library( deps = [dep + "_proto" for dep in proto[1][1]], ) for proto in WELL_KNOWN_PROTO_MAP.items()] +[native_cc_proto_library( + name = proto + "_cc_proto", + deps = [proto + "_proto"], + visibility = ["//visibility:private"], +) for proto in WELL_KNOWN_PROTO_MAP.keys()] + +cc_proto_blacklist_test( + name = "cc_proto_blacklist_test", + deps = [proto + "_cc_proto" for proto in WELL_KNOWN_PROTO_MAP.keys()] +) + ################################################################################ # Protocol Buffers Compiler ################################################################################ @@ -989,7 +1001,7 @@ cc_library( # Note: We use `native_proto_common` here because we depend on an implementation-detail of # `proto_lang_toolchain`, which may not be available on `proto_common`. -reject_blacklisted_files = not hasattr(native_proto_common, "proto_lang_toolchain_rejects_files_do_not_use_or_we_will_break_you_without_mercy") +reject_blacklisted_files = hasattr(native_proto_common, "proto_lang_toolchain_rejects_files_do_not_use_or_we_will_break_you_without_mercy") cc_toolchain_blacklisted_protos = [proto + "_proto" for proto in WELL_KNOWN_PROTO_MAP.keys()] if reject_blacklisted_files else [":well_known_protos"] proto_lang_toolchain( name = "cc_toolchain", diff --git a/WORKSPACE b/WORKSPACE index aedc4e0729..39672c8828 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -76,3 +76,8 @@ bind( name = "error_prone_annotations", actual = "@error_prone_annotations_maven//jar", ) + +# For `cc_proto_blacklist_test`. +load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") + +bazel_skylib_workspace() diff --git a/cc_proto_blacklist_test.bzl b/cc_proto_blacklist_test.bzl new file mode 100644 index 0000000000..df54293cbd --- /dev/null +++ b/cc_proto_blacklist_test.bzl @@ -0,0 +1,38 @@ +"""Contains a unittest to verify that `cc_proto_library` does not generate code for blacklisted `.proto` sources (i.e. WKPs).""" + +load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") + +def _cc_proto_blacklist_test_impl(ctx): + """Verifies that there are no C++ compile actions for Well-Known-Protos. + + Args: + ctx: The rule context. + + Returns: A (not further specified) sequence of providers. + """ + + env = unittest.begin(ctx) + + for dep in ctx.attr.deps: + files = len(dep.files.to_list()) + asserts.equals( + env, + 0, + files, + "Expected that target '{}' does not provide files, got {}".format( + dep.label, + files, + ), + ) + + return unittest.end(env) + +cc_proto_blacklist_test = unittest.make( + impl = _cc_proto_blacklist_test_impl, + attrs = { + "deps": attr.label_list( + mandatory = True, + providers = [CcInfo], + ), + }, +) diff --git a/kokoro/linux/bazel/build.sh b/kokoro/linux/bazel/build.sh index b99b4d31df..b5991d773a 100755 --- a/kokoro/linux/bazel/build.sh +++ b/kokoro/linux/bazel/build.sh @@ -23,7 +23,10 @@ cd $(dirname $0)/../../.. git submodule update --init --recursive trap print_test_logs EXIT -bazel test :build_files_updated_unittest :protobuf_test --copt=-Werror --host_copt=-Werror +bazel test --copt=-Werror --host_copt=-Werror \ + //:build_files_updated_unittest \ + //:protobuf_test \ + @com_google_protobuf//:cc_proto_blacklist_test trap - EXIT cd examples diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index 9209b1c5a3..fb0c47786d 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -8,9 +8,11 @@ def protobuf_deps(): if not native.existing_rule("bazel_skylib"): http_archive( name = "bazel_skylib", - sha256 = "bbccf674aa441c266df9894182d80de104cabd19be98be002f6d478aaa31574d", - strip_prefix = "bazel-skylib-2169ae1c374aab4a09aa90e65efe1a3aad4e279b", - urls = ["https://github.com/bazelbuild/bazel-skylib/archive/2169ae1c374aab4a09aa90e65efe1a3aad4e279b.tar.gz"], + sha256 = "97e70364e9249702246c0e9444bccdc4b847bed1eb03c5a3ece4f83dfe6abc44", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz", + ], ) if not native.existing_rule("zlib"): From 629ca244a102593ee5c5a42ab2dedc50d64e77ed Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Tue, 21 Jan 2020 15:38:42 -0800 Subject: [PATCH 05/27] Call register_class before getClass from desc (#7077) * Call register_class before getClass from desc * Remove debug code --- php/ext/google/protobuf/def.c | 1 + php/tests/descriptors_test.php | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/php/ext/google/protobuf/def.c b/php/ext/google/protobuf/def.c index f10c37cd9b..7f5004ced1 100644 --- a/php/ext/google/protobuf/def.c +++ b/php/ext/google/protobuf/def.c @@ -132,6 +132,7 @@ static void descriptor_init_c_instance(Descriptor *desc TSRMLS_DC) { PHP_METHOD(Descriptor, getClass) { Descriptor* desc = UNBOX(Descriptor, getThis()); DescriptorInternal* intern = desc->intern; + register_class(intern, false TSRMLS_CC); #if PHP_MAJOR_VERSION < 7 const char* classname = intern->klass->name; #else diff --git a/php/tests/descriptors_test.php b/php/tests/descriptors_test.php index 93683b82fd..60a6292cb6 100644 --- a/php/tests/descriptors_test.php +++ b/php/tests/descriptors_test.php @@ -11,6 +11,8 @@ use Google\Protobuf\Internal\MapField; use Descriptors\TestDescriptorsEnum; use Descriptors\TestDescriptorsMessage; use Descriptors\TestDescriptorsMessage\Sub; +use Foo\TestMessage; +use Bar\TestInclude; class DescriptorsTest extends TestBase { @@ -89,6 +91,17 @@ class DescriptorsTest extends TestBase $this->assertSame(1, $desc->getOneofDeclCount()); } + public function testDescriptorForIncludedMessage() + { + $pool = DescriptorPool::getGeneratedPool(); + $class = get_class(new TestMessage()); + $this->assertSame('Foo\TestMessage', $class); + $desc = $pool->getDescriptorByClassName($class); + $fielddesc = $desc->getField(17); + $subdesc = $fielddesc->getMessageType(); + $this->assertSame('Bar\TestInclude', $subdesc->getClass()); + } + ######################################################### # Test enum descriptor. ######################################################### From e8016753e3817d44d78806d5ab43b5482cba6efa Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Wed, 22 Jan 2020 12:29:56 -0800 Subject: [PATCH 06/27] Maven requires https connection (#7110) (#7114) --- csharp/compatibility_tests/v3.0.0/test.sh | 2 +- php/tests/compatibility_test.sh | 2 +- python/compatibility_tests/v2.5.0/test.sh | 2 +- ruby/compatibility_tests/v3.0.0/test.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/csharp/compatibility_tests/v3.0.0/test.sh b/csharp/compatibility_tests/v3.0.0/test.sh index 4ee88fc8e9..cab215a314 100755 --- a/csharp/compatibility_tests/v3.0.0/test.sh +++ b/csharp/compatibility_tests/v3.0.0/test.sh @@ -40,7 +40,7 @@ TEST_VERSION=3.0.0 # is also possible to use this same test set to test the compatibiilty of the # latest version against other versions. OLD_VERSION=$1 -OLD_VERSION_PROTOC=http://repo1.maven.org/maven2/com/google/protobuf/protoc/$OLD_VERSION/protoc-$OLD_VERSION-linux-x86_64.exe +OLD_VERSION_PROTOC=https://repo1.maven.org/maven2/com/google/protobuf/protoc/$OLD_VERSION/protoc-$OLD_VERSION-linux-x86_64.exe echo "Running compatibility tests with $OLD_VERSION" diff --git a/php/tests/compatibility_test.sh b/php/tests/compatibility_test.sh index 8e1a75092b..2b5c6ab5b7 100755 --- a/php/tests/compatibility_test.sh +++ b/php/tests/compatibility_test.sh @@ -71,7 +71,7 @@ set -ex cd $(dirname $0) OLD_VERSION=$1 -OLD_VERSION_PROTOC=http://repo1.maven.org/maven2/com/google/protobuf/protoc/$OLD_VERSION/protoc-$OLD_VERSION-linux-x86_64.exe +OLD_VERSION_PROTOC=https://repo1.maven.org/maven2/com/google/protobuf/protoc/$OLD_VERSION/protoc-$OLD_VERSION-linux-x86_64.exe # Extract the latest protobuf version number. VERSION_NUMBER=`grep "PHP_PROTOBUF_VERSION" ../ext/google/protobuf/protobuf.h | sed "s|#define PHP_PROTOBUF_VERSION \"\(.*\)\"|\1|"` diff --git a/python/compatibility_tests/v2.5.0/test.sh b/python/compatibility_tests/v2.5.0/test.sh index fb3e545f41..b5a170c21c 100755 --- a/python/compatibility_tests/v2.5.0/test.sh +++ b/python/compatibility_tests/v2.5.0/test.sh @@ -15,7 +15,7 @@ TEST_VERSION=2.5.0 # is also possible to use this same test set to test the compatibiilty of the # latest version against other versions. OLD_VERSION=$1 -OLD_VERSION_PROTOC=http://repo1.maven.org/maven2/com/google/protobuf/protoc/$OLD_VERSION/protoc-$OLD_VERSION-linux-x86_64.exe +OLD_VERSION_PROTOC=https://repo1.maven.org/maven2/com/google/protobuf/protoc/$OLD_VERSION/protoc-$OLD_VERSION-linux-x86_64.exe # Extract the latest protobuf version number. VERSION_NUMBER=`grep "^__version__ = '.*'" ../../google/protobuf/__init__.py | sed "s|__version__ = '\(.*\)'|\1|"` diff --git a/ruby/compatibility_tests/v3.0.0/test.sh b/ruby/compatibility_tests/v3.0.0/test.sh index 996dc020c2..d5a8fd74d7 100755 --- a/ruby/compatibility_tests/v3.0.0/test.sh +++ b/ruby/compatibility_tests/v3.0.0/test.sh @@ -10,7 +10,7 @@ PROTOC_BINARY_NAME="protoc-3.0.0-linux-x86_64.exe" if [ `uname` = "Darwin" ]; then PROTOC_BINARY_NAME="protoc-3.0.0-osx-x86_64.exe" fi -wget http://repo1.maven.org/maven2/com/google/protobuf/protoc/3.0.0/${PROTOC_BINARY_NAME} -O protoc +wget https://repo1.maven.org/maven2/com/google/protobuf/protoc/3.0.0/${PROTOC_BINARY_NAME} -O protoc chmod +x protoc # Run tests From 85219578fad77fd7d006d4c8efe9de48791b895f Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Thu, 30 Jan 2020 12:45:47 -0800 Subject: [PATCH 07/27] Add continuous test for php7.4 on mac (#7153) --- kokoro/macos/php7.4_mac/build.sh | 11 +++++++++++ kokoro/macos/php7.4_mac/continuous.cfg | 5 +++++ kokoro/macos/php7.4_mac/presubmit.cfg | 5 +++++ tests.sh | 24 ++++++++++++++++++++++++ 4 files changed, 45 insertions(+) create mode 100755 kokoro/macos/php7.4_mac/build.sh create mode 100644 kokoro/macos/php7.4_mac/continuous.cfg create mode 100644 kokoro/macos/php7.4_mac/presubmit.cfg diff --git a/kokoro/macos/php7.4_mac/build.sh b/kokoro/macos/php7.4_mac/build.sh new file mode 100755 index 0000000000..98c82d4bb3 --- /dev/null +++ b/kokoro/macos/php7.4_mac/build.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# +# Build file to set up and run tests + +# Change to repo root +cd $(dirname $0)/../../.. + +# Prepare worker environment to run tests +source kokoro/macos/prepare_build_macos_rc + +./tests.sh php7.4_mac diff --git a/kokoro/macos/php7.4_mac/continuous.cfg b/kokoro/macos/php7.4_mac/continuous.cfg new file mode 100644 index 0000000000..5b2d6fde98 --- /dev/null +++ b/kokoro/macos/php7.4_mac/continuous.cfg @@ -0,0 +1,5 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/macos/php7.4_mac/build.sh" +timeout_mins: 1440 diff --git a/kokoro/macos/php7.4_mac/presubmit.cfg b/kokoro/macos/php7.4_mac/presubmit.cfg new file mode 100644 index 0000000000..5b2d6fde98 --- /dev/null +++ b/kokoro/macos/php7.4_mac/presubmit.cfg @@ -0,0 +1,5 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/macos/php7.4_mac/build.sh" +timeout_mins: 1440 diff --git a/tests.sh b/tests.sh index 4c7a802324..8785439228 100755 --- a/tests.sh +++ b/tests.sh @@ -715,6 +715,30 @@ build_php7.0_mac() { popd } +build_php7.4_mac() { + generate_php_test_proto + # Install PHP + curl -s https://php-osx.liip.ch/install.sh | bash -s 7.4 + PHP_FOLDER=`find /usr/local -type d -name "php7-7.4*"` # The folder name may change upon time + export PATH="$PHP_FOLDER/bin:$PATH" + + # Install phpunit + curl https://phar.phpunit.de/phpunit-8.phar -L -o phpunit.phar + chmod +x phpunit.phar + sudo mv phpunit.phar /usr/local/bin/phpunit + + # Install valgrind + echo "#! /bin/bash" > valgrind + chmod ug+x valgrind + sudo mv valgrind /usr/local/bin/valgrind + + # Test + cd php/tests && /bin/bash ./test.sh && cd ../.. + pushd conformance + make test_php_c + popd +} + build_php_compatibility() { internal_build_cpp php/tests/compatibility_test.sh $LAST_RELEASED From c5eada8eeb88f13e7f353df7dc9c04f969ce6ba6 Mon Sep 17 00:00:00 2001 From: Luca Santarella Date: Thu, 30 Jan 2020 19:47:32 -0500 Subject: [PATCH 08/27] Refactored ulong to zend_ulong for php7.4 compatibility (#7147) --- php/ext/google/protobuf/protobuf.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/php/ext/google/protobuf/protobuf.h b/php/ext/google/protobuf/protobuf.h index 4e6d56a358..04ace6a2f4 100644 --- a/php/ext/google/protobuf/protobuf.h +++ b/php/ext/google/protobuf/protobuf.h @@ -292,7 +292,7 @@ #define PHP_PROTO_HASH_OF(array) Z_ARRVAL_P(&array) -static inline int php_proto_zend_hash_index_update_zval(HashTable* ht, ulong h, +static inline int php_proto_zend_hash_index_update_zval(HashTable* ht, zend_ulong h, zval* pData) { void* result = NULL; result = zend_hash_index_update(ht, h, pData); @@ -308,7 +308,7 @@ static inline int php_proto_zend_hash_update(HashTable* ht, const char* key, return result != NULL ? SUCCESS : FAILURE; } -static inline int php_proto_zend_hash_index_update_mem(HashTable* ht, ulong h, +static inline int php_proto_zend_hash_index_update_mem(HashTable* ht, zend_ulong h, void* pData, uint nDataSize, void** pDest) { void* result = NULL; @@ -337,7 +337,7 @@ static inline int php_proto_zend_hash_update_mem(HashTable* ht, const char* key, } static inline int php_proto_zend_hash_index_find_zval(const HashTable* ht, - ulong h, void** pDest) { + zend_ulong h, void** pDest) { zval* result = zend_hash_index_find(ht, h); if (pDest != NULL) *pDest = result; return result != NULL ? SUCCESS : FAILURE; @@ -351,7 +351,7 @@ static inline int php_proto_zend_hash_find(const HashTable* ht, const char* key, } static inline int php_proto_zend_hash_index_find_mem(const HashTable* ht, - ulong h, void** pDest) { + zend_ulong h, void** pDest) { void* result = NULL; result = zend_hash_index_find_ptr(ht, h); if (pDest != NULL) *pDest = result; From 47434eb43d0e3e77f3faeb7e23efd6d1221b8481 Mon Sep 17 00:00:00 2001 From: Elliotte Rusty Harold Date: Fri, 31 Jan 2020 12:43:25 -0500 Subject: [PATCH 09/27] deps: update errorprone to 2.3.4 (#7125) (#7157) --- java/util/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/util/pom.xml b/java/util/pom.xml index c4f0e5499d..56d648884a 100644 --- a/java/util/pom.xml +++ b/java/util/pom.xml @@ -25,7 +25,7 @@ com.google.errorprone error_prone_annotations - 2.3.3 + 2.3.4 com.google.guava From bc237eb67268181bc8c152539407430e12e98557 Mon Sep 17 00:00:00 2001 From: Rafi Kamal Date: Fri, 31 Jan 2020 09:43:58 -0800 Subject: [PATCH 10/27] Install RubyGem bundler version specified in Gemfile.lock to fix release failure (#7144) (#7156) Ref: https://bundler.io/blog/2019/05/14/solutions-for-cant-find-gem-bundler-with-executable-bundle.html --- kokoro/release/ruby/linux/ruby/ruby_build_environment.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kokoro/release/ruby/linux/ruby/ruby_build_environment.sh b/kokoro/release/ruby/linux/ruby/ruby_build_environment.sh index ea04f9055c..87c0f75448 100755 --- a/kokoro/release/ruby/linux/ruby/ruby_build_environment.sh +++ b/kokoro/release/ruby/linux/ruby/ruby_build_environment.sh @@ -4,5 +4,6 @@ set +ex [[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh set -e # rvm commands are very verbose rvm --default use ruby-2.4.1 -gem install bundler --update +# The version needs to be updated if the version specified in Gemfile.lock is changed +gem install bundler -v '1.17.3' set -ex From 345df38dd349ebfc4e8bfeefb8de249633f39026 Mon Sep 17 00:00:00 2001 From: Rafi Kamal Date: Fri, 31 Jan 2020 13:47:09 -0800 Subject: [PATCH 11/27] Update protobuf version (#7143) --- Protobuf-C++.podspec | 2 +- Protobuf.podspec | 2 +- configure.ac | 2 +- csharp/Google.Protobuf.Tools.nuspec | 2 +- .../Google.Protobuf/Google.Protobuf.csproj | 2 +- java/bom/pom.xml | 2 +- java/core/pom.xml | 2 +- java/lite/pom.xml | 2 +- java/pom.xml | 2 +- java/util/pom.xml | 2 +- js/package.json | 2 +- php/ext/google/protobuf/package.xml | 22 +++++++++++++++---- php/ext/google/protobuf/protobuf.h | 2 +- protoc-artifacts/pom.xml | 2 +- python/google/protobuf/__init__.py | 2 +- ruby/google-protobuf.gemspec | 2 +- src/Makefile.am | 2 +- src/google/protobuf/any.pb.h | 2 +- src/google/protobuf/api.pb.h | 2 +- src/google/protobuf/compiler/plugin.pb.h | 2 +- src/google/protobuf/descriptor.pb.h | 2 +- src/google/protobuf/duration.pb.h | 2 +- src/google/protobuf/empty.pb.h | 2 +- src/google/protobuf/field_mask.pb.h | 2 +- src/google/protobuf/port_def.inc | 2 +- src/google/protobuf/source_context.pb.h | 2 +- src/google/protobuf/struct.pb.h | 2 +- src/google/protobuf/stubs/common.h | 2 +- src/google/protobuf/timestamp.pb.h | 2 +- src/google/protobuf/type.pb.h | 2 +- src/google/protobuf/wrappers.pb.h | 2 +- 31 files changed, 48 insertions(+), 34 deletions(-) diff --git a/Protobuf-C++.podspec b/Protobuf-C++.podspec index 1359d0294d..db3857765f 100644 --- a/Protobuf-C++.podspec +++ b/Protobuf-C++.podspec @@ -1,6 +1,6 @@ Pod::Spec.new do |s| s.name = 'Protobuf-C++' - s.version = '3.11.2' + s.version = '3.11.3' s.summary = 'Protocol Buffers v3 runtime library for C++.' s.homepage = 'https://github.com/google/protobuf' s.license = '3-Clause BSD License' diff --git a/Protobuf.podspec b/Protobuf.podspec index 6de36e05ac..73d40eee25 100644 --- a/Protobuf.podspec +++ b/Protobuf.podspec @@ -5,7 +5,7 @@ # dependent projects use the :git notation to refer to the library. Pod::Spec.new do |s| s.name = 'Protobuf' - s.version = '3.11.2' + s.version = '3.11.3' s.summary = 'Protocol Buffers v.3 runtime library for Objective-C.' s.homepage = 'https://github.com/protocolbuffers/protobuf' s.license = '3-Clause BSD License' diff --git a/configure.ac b/configure.ac index df39c05ea5..bf51d8b670 100644 --- a/configure.ac +++ b/configure.ac @@ -17,7 +17,7 @@ AC_PREREQ(2.59) # In the SVN trunk, the version should always be the next anticipated release # version with the "-pre" suffix. (We used to use "-SNAPSHOT" but this pushed # the size of one file name in the dist tarfile over the 99-char limit.) -AC_INIT([Protocol Buffers],[3.11.2],[protobuf@googlegroups.com],[protobuf]) +AC_INIT([Protocol Buffers],[3.11.3],[protobuf@googlegroups.com],[protobuf]) AM_MAINTAINER_MODE([enable]) diff --git a/csharp/Google.Protobuf.Tools.nuspec b/csharp/Google.Protobuf.Tools.nuspec index 3823e20583..ff983da9b7 100644 --- a/csharp/Google.Protobuf.Tools.nuspec +++ b/csharp/Google.Protobuf.Tools.nuspec @@ -5,7 +5,7 @@ Google Protocol Buffers tools Tools for Protocol Buffers - Google's data interchange format. See project site for more info. - 3.11.2 + 3.11.3 Google Inc. protobuf-packages https://github.com/protocolbuffers/protobuf/blob/master/LICENSE diff --git a/csharp/src/Google.Protobuf/Google.Protobuf.csproj b/csharp/src/Google.Protobuf/Google.Protobuf.csproj index e8e33e27e6..2cc7d2373f 100644 --- a/csharp/src/Google.Protobuf/Google.Protobuf.csproj +++ b/csharp/src/Google.Protobuf/Google.Protobuf.csproj @@ -4,7 +4,7 @@ C# runtime library for Protocol Buffers - Google's data interchange format. Copyright 2015, Google Inc. Google Protocol Buffers - 3.11.2 + 3.11.3 6 Google Inc. netstandard1.0;netstandard2.0;net45 diff --git a/java/bom/pom.xml b/java/bom/pom.xml index ea15d48190..d5c9f07e76 100644 --- a/java/bom/pom.xml +++ b/java/bom/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-bom - 3.11.2 + 3.11.3 pom Protocol Buffers [BOM] diff --git a/java/core/pom.xml b/java/core/pom.xml index a57f2aa43c..14145f733c 100644 --- a/java/core/pom.xml +++ b/java/core/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.11.2 + 3.11.3 protobuf-java diff --git a/java/lite/pom.xml b/java/lite/pom.xml index 81e052104f..8504f82d69 100644 --- a/java/lite/pom.xml +++ b/java/lite/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.11.2 + 3.11.3 protobuf-javalite diff --git a/java/pom.xml b/java/pom.xml index 89aeb632c5..6eda963353 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.11.2 + 3.11.3 pom Protocol Buffers [Parent] diff --git a/java/util/pom.xml b/java/util/pom.xml index 56d648884a..67b802b051 100644 --- a/java/util/pom.xml +++ b/java/util/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.11.2 + 3.11.3 protobuf-java-util diff --git a/js/package.json b/js/package.json index a2fb371742..942beb55ca 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "google-protobuf", - "version": "3.11.2", + "version": "3.11.3", "description": "Protocol Buffers for JavaScript", "main": "google-protobuf.js", "files": [ diff --git a/php/ext/google/protobuf/package.xml b/php/ext/google/protobuf/package.xml index 99080b750f..d5daa7b078 100644 --- a/php/ext/google/protobuf/package.xml +++ b/php/ext/google/protobuf/package.xml @@ -10,11 +10,11 @@ protobuf-opensource@google.com yes - 2019-12-10 - + 2020-01-28 + - 3.11.2 - 3.11.2 + 3.11.3 + 3.11.3 stable @@ -501,5 +501,19 @@ G A release. 3-Clause BSD License GA release. + + + 3.11.3 + 3.11.3 + + + stable + stable + + 2020-01-28 + + 3-Clause BSD License + GA release. + diff --git a/php/ext/google/protobuf/protobuf.h b/php/ext/google/protobuf/protobuf.h index 04ace6a2f4..f9c1c792f4 100644 --- a/php/ext/google/protobuf/protobuf.h +++ b/php/ext/google/protobuf/protobuf.h @@ -37,7 +37,7 @@ #include "upb.h" #define PHP_PROTOBUF_EXTNAME "protobuf" -#define PHP_PROTOBUF_VERSION "3.11.2" +#define PHP_PROTOBUF_VERSION "3.11.3" #define MAX_LENGTH_OF_INT64 20 #define SIZEOF_INT64 8 diff --git a/protoc-artifacts/pom.xml b/protoc-artifacts/pom.xml index 924fb5d42b..3327802099 100644 --- a/protoc-artifacts/pom.xml +++ b/protoc-artifacts/pom.xml @@ -8,7 +8,7 @@ com.google.protobuf protoc - 3.11.2 + 3.11.3 pom Protobuf Compiler diff --git a/python/google/protobuf/__init__.py b/python/google/protobuf/__init__.py index 25d2445ff9..c0745d0e7a 100755 --- a/python/google/protobuf/__init__.py +++ b/python/google/protobuf/__init__.py @@ -30,7 +30,7 @@ # Copyright 2007 Google Inc. All Rights Reserved. -__version__ = '3.11.2' +__version__ = '3.11.3' if __name__ != '__main__': try: diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index ba7acbc2ac..8f51cf65a8 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = "google-protobuf" - s.version = "3.11.2" + s.version = "3.11.3" git_tag = "v#{s.version.to_s.sub('.rc.', '-rc')}" # Converts X.Y.Z.rc.N to vX.Y.Z-rcN, used for the git tag s.licenses = ["BSD-3-Clause"] s.summary = "Protocol Buffers" diff --git a/src/Makefile.am b/src/Makefile.am index 5ac489b6bc..9d7bc4c126 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -18,7 +18,7 @@ else PTHREAD_DEF = endif -PROTOBUF_VERSION = 22:2:0 +PROTOBUF_VERSION = 22:3:0 if GCC # Turn on all warnings except for sign comparison (we ignore sign comparison diff --git a/src/google/protobuf/any.pb.h b/src/google/protobuf/any.pb.h index f551818e61..87c2aedfd0 100644 --- a/src/google/protobuf/any.pb.h +++ b/src/google/protobuf/any.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/api.pb.h b/src/google/protobuf/api.pb.h index b00b88dc24..e1f32a641c 100644 --- a/src/google/protobuf/api.pb.h +++ b/src/google/protobuf/api.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/compiler/plugin.pb.h b/src/google/protobuf/compiler/plugin.pb.h index 0fdf8c6ec9..0040463fce 100644 --- a/src/google/protobuf/compiler/plugin.pb.h +++ b/src/google/protobuf/compiler/plugin.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/descriptor.pb.h b/src/google/protobuf/descriptor.pb.h index 30e85cca72..48ef8a1863 100644 --- a/src/google/protobuf/descriptor.pb.h +++ b/src/google/protobuf/descriptor.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/duration.pb.h b/src/google/protobuf/duration.pb.h index 7b34a4bf02..33af624bc1 100644 --- a/src/google/protobuf/duration.pb.h +++ b/src/google/protobuf/duration.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/empty.pb.h b/src/google/protobuf/empty.pb.h index 66312bc1cc..2c22bd3509 100644 --- a/src/google/protobuf/empty.pb.h +++ b/src/google/protobuf/empty.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/field_mask.pb.h b/src/google/protobuf/field_mask.pb.h index f3f83194d8..24c5c44782 100644 --- a/src/google/protobuf/field_mask.pb.h +++ b/src/google/protobuf/field_mask.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 3e39f915f6..375f919d70 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -297,7 +297,7 @@ // Shared google3/opensource definitions. ////////////////////////////////////// -#define PROTOBUF_VERSION 3011002 +#define PROTOBUF_VERSION 3011003 #define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC 3011000 #define PROTOBUF_MIN_PROTOC_VERSION 3011000 #define PROTOBUF_VERSION_SUFFIX "" diff --git a/src/google/protobuf/source_context.pb.h b/src/google/protobuf/source_context.pb.h index c9c70c8052..77349c8a6e 100644 --- a/src/google/protobuf/source_context.pb.h +++ b/src/google/protobuf/source_context.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/struct.pb.h b/src/google/protobuf/struct.pb.h index 9d2ccb2016..09306bc817 100644 --- a/src/google/protobuf/struct.pb.h +++ b/src/google/protobuf/struct.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/stubs/common.h b/src/google/protobuf/stubs/common.h index 3301a4f1b8..453f6313f0 100644 --- a/src/google/protobuf/stubs/common.h +++ b/src/google/protobuf/stubs/common.h @@ -81,7 +81,7 @@ namespace internal { // The current version, represented as a single integer to make comparison // easier: major * 10^6 + minor * 10^3 + micro -#define GOOGLE_PROTOBUF_VERSION 3011002 +#define GOOGLE_PROTOBUF_VERSION 3011003 // A suffix string for alpha, beta or rc releases. Empty for stable releases. #define GOOGLE_PROTOBUF_VERSION_SUFFIX "" diff --git a/src/google/protobuf/timestamp.pb.h b/src/google/protobuf/timestamp.pb.h index 338a1e707b..bae12ebac4 100644 --- a/src/google/protobuf/timestamp.pb.h +++ b/src/google/protobuf/timestamp.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/type.pb.h b/src/google/protobuf/type.pb.h index 40d2cd4f0f..074d10e668 100644 --- a/src/google/protobuf/type.pb.h +++ b/src/google/protobuf/type.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/wrappers.pb.h b/src/google/protobuf/wrappers.pb.h index ebde03a63f..d62d120eb6 100644 --- a/src/google/protobuf/wrappers.pb.h +++ b/src/google/protobuf/wrappers.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011002 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. From 5f3d6599f61a01790d6682fe4e3f2c81c3ae09d9 Mon Sep 17 00:00:00 2001 From: Rafi Kamal Date: Fri, 31 Jan 2020 16:00:28 -0800 Subject: [PATCH 12/27] Update CHANGES.txt for 3.11.3 release (#7162) --- CHANGES.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index fd65e60deb..b047b97208 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,13 @@ +2020-01-31 version 3.11.3 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) + + C++ + * Add OUT and OPTIONAL to windows portability files (#7087) + + PHP + * Refactored ulong to zend_ulong for php7.4 compatibility (#7147) + * Call register_class before getClass from desc to fix segfault (#7077) + + 2019-12-10 version 3.11.2 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) PHP From 498de9f761bef56a032815ee44b6e6dbe0892cc4 Mon Sep 17 00:00:00 2001 From: Rafi Kamal Date: Sun, 2 Feb 2020 14:04:32 -0800 Subject: [PATCH 13/27] Changed the maven URL to use https when building dist artifacts --- protoc-artifacts/build-zip.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protoc-artifacts/build-zip.sh b/protoc-artifacts/build-zip.sh index 392bc1491f..7c4daf47ee 100755 --- a/protoc-artifacts/build-zip.sh +++ b/protoc-artifacts/build-zip.sh @@ -100,7 +100,7 @@ for((i=0;i<${#FILE_NAMES[@]};i+=2));do BINARY="$TARGET" fi BINARY_NAME=${FILE_NAMES[$(($i+1))]} - BINARY_URL=http://repo1.maven.org/maven2/com/google/protobuf/$TARGET/${VERSION_NUMBER}/$TARGET-${VERSION_NUMBER}-${BINARY_NAME} + BINARY_URL=https://repo1.maven.org/maven2/com/google/protobuf/$TARGET/${VERSION_NUMBER}/$TARGET-${VERSION_NUMBER}-${BINARY_NAME} if ! wget ${BINARY_URL} -O ${DIR}/bin/$BINARY &> /dev/null; then echo "[ERROR] Failed to download ${BINARY_URL}" >&2 echo "[ERROR] Skipped $TARGET-${VERSION_NAME}-${ZIP_NAME}" >&2 From fedb2beee3f959d2a80bfda0a76b2613efb8fc1c Mon Sep 17 00:00:00 2001 From: Rafi Kamal Date: Mon, 10 Feb 2020 10:46:19 -0800 Subject: [PATCH 14/27] Merge pull request #6938 from ObsidianMinor/csharp/fix/6936 (#7188) Fix latest ArgumentException for C# extensions Co-authored-by: Jan Tattermusch --- Makefile.am | 6 + csharp/generate_protos.sh | 3 + csharp/protos/unittest_issue6936_a.proto | 15 ++ csharp/protos/unittest_issue6936_b.proto | 14 ++ csharp/protos/unittest_issue6936_c.proto | 16 ++ .../UnittestIssue6936A.cs | 46 +++++ .../UnittestIssue6936B.cs | 145 ++++++++++++++ .../UnittestIssue6936C.cs | 181 ++++++++++++++++++ .../Reflection/CustomOptionsTest.cs | 10 + csharp/src/Google.Protobuf.Test/testprotos.pb | Bin 329562 -> 330948 bytes .../src/Google.Protobuf/ExtensionRegistry.cs | 13 ++ .../Reflection/FileDescriptor.cs | 7 +- 12 files changed, 453 insertions(+), 3 deletions(-) create mode 100644 csharp/protos/unittest_issue6936_a.proto create mode 100644 csharp/protos/unittest_issue6936_b.proto create mode 100644 csharp/protos/unittest_issue6936_c.proto create mode 100644 csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936A.cs create mode 100644 csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936B.cs create mode 100644 csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936C.cs diff --git a/Makefile.am b/Makefile.am index ed852c8d51..cca346e9ca 100644 --- a/Makefile.am +++ b/Makefile.am @@ -68,6 +68,9 @@ csharp_EXTRA_DIST= \ csharp/protos/map_unittest_proto3.proto \ csharp/protos/old_extensions1.proto \ csharp/protos/old_extensions2.proto \ + csharp/protos/unittest_issue6936_a.proto \ + csharp/protos/unittest_issue6936_b.proto \ + csharp/protos/unittest_issue6936_c.proto \ csharp/protos/unittest_custom_options_proto3.proto \ csharp/protos/unittest_import_public_proto3.proto \ csharp/protos/unittest_import_public.proto \ @@ -128,6 +131,9 @@ csharp_EXTRA_DIST= \ csharp/src/Google.Protobuf.Test/SampleNaNs.cs \ csharp/src/Google.Protobuf.Test/TestCornerCases.cs \ csharp/src/Google.Protobuf.Test.TestProtos/Google.Protobuf.Test.TestProtos.csproj \ + csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936A.cs \ + csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936B.cs \ + csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936C.cs \ csharp/src/Google.Protobuf.Test.TestProtos/ForeignMessagePartial.cs \ csharp/src/Google.Protobuf.Test.TestProtos/MapUnittestProto3.cs \ csharp/src/Google.Protobuf.Test.TestProtos/OldExtensions1.cs \ diff --git a/csharp/generate_protos.sh b/csharp/generate_protos.sh index 66d87c02f7..e6687c308e 100755 --- a/csharp/generate_protos.sh +++ b/csharp/generate_protos.sh @@ -58,6 +58,9 @@ $PROTOC -Isrc -Icsharp/protos \ csharp/protos/unittest.proto \ csharp/protos/unittest_import.proto \ csharp/protos/unittest_import_public.proto \ + csharp/protos/unittest_issue6936_a.proto \ + csharp/protos/unittest_issue6936_b.proto \ + csharp/protos/unittest_issue6936_c.proto \ src/google/protobuf/unittest_well_known_types.proto \ src/google/protobuf/test_messages_proto3.proto \ src/google/protobuf/test_messages_proto2.proto diff --git a/csharp/protos/unittest_issue6936_a.proto b/csharp/protos/unittest_issue6936_a.proto new file mode 100644 index 0000000000..097d083a98 --- /dev/null +++ b/csharp/protos/unittest_issue6936_a.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package unittest_issues; + +option csharp_namespace = "UnitTest.Issues.TestProtos"; + +// This file is used as part of a unit test for issue 6936 +// We don't need to use it, we just have to import it in both +// "extensions_issue6936_b.proto" and "extensions_issue6936_c.proto" + +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.MessageOptions { + string opt = 50000; +} \ No newline at end of file diff --git a/csharp/protos/unittest_issue6936_b.proto b/csharp/protos/unittest_issue6936_b.proto new file mode 100644 index 0000000000..8f71683ba6 --- /dev/null +++ b/csharp/protos/unittest_issue6936_b.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +import "unittest_issue6936_a.proto"; + +package unittest_issues; + +option csharp_namespace = "UnitTest.Issues.TestProtos"; + +// This file is used as part of a unit test for issue 6936 +// We don't need to use it, we just have to import it in "unittest_issue6936_c.proto" + +message Foo { + option (opt) = "foo"; +} \ No newline at end of file diff --git a/csharp/protos/unittest_issue6936_c.proto b/csharp/protos/unittest_issue6936_c.proto new file mode 100644 index 0000000000..40004ecaae --- /dev/null +++ b/csharp/protos/unittest_issue6936_c.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +import "unittest_issue6936_a.proto"; +import "unittest_issue6936_b.proto"; + +package unittest_issues; + +option csharp_namespace = "UnitTest.Issues.TestProtos"; + +// This file is used as part of a unit test for issue 6936 +// We don't need to use it, we just have to load it at runtime + +message Bar { + option (opt) = "bar"; + Foo foo = 1; +} \ No newline at end of file diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936A.cs b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936A.cs new file mode 100644 index 0000000000..56fde4f00a --- /dev/null +++ b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936A.cs @@ -0,0 +1,46 @@ +// +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: unittest_issue6936_a.proto +// +#pragma warning disable 1591, 0612, 3021 +#region Designer generated code + +using pb = global::Google.Protobuf; +using pbc = global::Google.Protobuf.Collections; +using pbr = global::Google.Protobuf.Reflection; +using scg = global::System.Collections.Generic; +namespace UnitTest.Issues.TestProtos { + + /// Holder for reflection information generated from unittest_issue6936_a.proto + public static partial class UnittestIssue6936AReflection { + + #region Descriptor + /// File descriptor for unittest_issue6936_a.proto + public static pbr::FileDescriptor Descriptor { + get { return descriptor; } + } + private static pbr::FileDescriptor descriptor; + + static UnittestIssue6936AReflection() { + byte[] descriptorData = global::System.Convert.FromBase64String( + string.Concat( + "Chp1bml0dGVzdF9pc3N1ZTY5MzZfYS5wcm90bxIPdW5pdHRlc3RfaXNzdWVz", + "GiBnb29nbGUvcHJvdG9idWYvZGVzY3JpcHRvci5wcm90bzouCgNvcHQSHy5n", + "b29nbGUucHJvdG9idWYuTWVzc2FnZU9wdGlvbnMY0IYDIAEoCUIdqgIaVW5p", + "dFRlc3QuSXNzdWVzLlRlc3RQcm90b3NiBnByb3RvMw==")); + descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, + new pbr::FileDescriptor[] { global::Google.Protobuf.Reflection.DescriptorReflection.Descriptor, }, + new pbr::GeneratedClrTypeInfo(null, new pb::Extension[] { UnittestIssue6936AExtensions.Opt }, null)); + } + #endregion + + } + /// Holder for extension identifiers generated from the top level of unittest_issue6936_a.proto + public static partial class UnittestIssue6936AExtensions { + public static readonly pb::Extension Opt = + new pb::Extension(50000, pb::FieldCodec.ForString(400002, "")); + } + +} + +#endregion Designer generated code diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936B.cs b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936B.cs new file mode 100644 index 0000000000..6b816f739b --- /dev/null +++ b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936B.cs @@ -0,0 +1,145 @@ +// +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: unittest_issue6936_b.proto +// +#pragma warning disable 1591, 0612, 3021 +#region Designer generated code + +using pb = global::Google.Protobuf; +using pbc = global::Google.Protobuf.Collections; +using pbr = global::Google.Protobuf.Reflection; +using scg = global::System.Collections.Generic; +namespace UnitTest.Issues.TestProtos { + + /// Holder for reflection information generated from unittest_issue6936_b.proto + public static partial class UnittestIssue6936BReflection { + + #region Descriptor + /// File descriptor for unittest_issue6936_b.proto + public static pbr::FileDescriptor Descriptor { + get { return descriptor; } + } + private static pbr::FileDescriptor descriptor; + + static UnittestIssue6936BReflection() { + byte[] descriptorData = global::System.Convert.FromBase64String( + string.Concat( + "Chp1bml0dGVzdF9pc3N1ZTY5MzZfYi5wcm90bxIPdW5pdHRlc3RfaXNzdWVz", + "Ghp1bml0dGVzdF9pc3N1ZTY5MzZfYS5wcm90byIOCgNGb286B4K1GANmb29C", + "HaoCGlVuaXRUZXN0Lklzc3Vlcy5UZXN0UHJvdG9zYgZwcm90bzM=")); + descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, + new pbr::FileDescriptor[] { global::UnitTest.Issues.TestProtos.UnittestIssue6936AReflection.Descriptor, }, + new pbr::GeneratedClrTypeInfo(null, null, new pbr::GeneratedClrTypeInfo[] { + new pbr::GeneratedClrTypeInfo(typeof(global::UnitTest.Issues.TestProtos.Foo), global::UnitTest.Issues.TestProtos.Foo.Parser, null, null, null, null, null) + })); + } + #endregion + + } + #region Messages + public sealed partial class Foo : pb::IMessage { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new Foo()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public static pbr::MessageDescriptor Descriptor { + get { return global::UnitTest.Issues.TestProtos.UnittestIssue6936BReflection.Descriptor.MessageTypes[0]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public Foo() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public Foo(Foo other) : this() { + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public Foo Clone() { + return new Foo(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override bool Equals(object other) { + return Equals(other as Foo); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public bool Equals(Foo other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override int GetHashCode() { + int hash = 1; + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void WriteTo(pb::CodedOutputStream output) { + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public int CalculateSize() { + int size = 0; + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void MergeFrom(Foo other) { + if (other == null) { + return; + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void MergeFrom(pb::CodedInputStream input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + } + } + } + + } + + #endregion + +} + +#endregion Designer generated code diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936C.cs b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936C.cs new file mode 100644 index 0000000000..a8a73f6cb6 --- /dev/null +++ b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936C.cs @@ -0,0 +1,181 @@ +// +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: unittest_issue6936_c.proto +// +#pragma warning disable 1591, 0612, 3021 +#region Designer generated code + +using pb = global::Google.Protobuf; +using pbc = global::Google.Protobuf.Collections; +using pbr = global::Google.Protobuf.Reflection; +using scg = global::System.Collections.Generic; +namespace UnitTest.Issues.TestProtos { + + /// Holder for reflection information generated from unittest_issue6936_c.proto + public static partial class UnittestIssue6936CReflection { + + #region Descriptor + /// File descriptor for unittest_issue6936_c.proto + public static pbr::FileDescriptor Descriptor { + get { return descriptor; } + } + private static pbr::FileDescriptor descriptor; + + static UnittestIssue6936CReflection() { + byte[] descriptorData = global::System.Convert.FromBase64String( + string.Concat( + "Chp1bml0dGVzdF9pc3N1ZTY5MzZfYy5wcm90bxIPdW5pdHRlc3RfaXNzdWVz", + "Ghp1bml0dGVzdF9pc3N1ZTY5MzZfYS5wcm90bxoadW5pdHRlc3RfaXNzdWU2", + "OTM2X2IucHJvdG8iMQoDQmFyEiEKA2ZvbxgBIAEoCzIULnVuaXR0ZXN0X2lz", + "c3Vlcy5Gb286B4K1GANiYXJCHaoCGlVuaXRUZXN0Lklzc3Vlcy5UZXN0UHJv", + "dG9zYgZwcm90bzM=")); + descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, + new pbr::FileDescriptor[] { global::UnitTest.Issues.TestProtos.UnittestIssue6936AReflection.Descriptor, global::UnitTest.Issues.TestProtos.UnittestIssue6936BReflection.Descriptor, }, + new pbr::GeneratedClrTypeInfo(null, null, new pbr::GeneratedClrTypeInfo[] { + new pbr::GeneratedClrTypeInfo(typeof(global::UnitTest.Issues.TestProtos.Bar), global::UnitTest.Issues.TestProtos.Bar.Parser, new[]{ "Foo" }, null, null, null, null) + })); + } + #endregion + + } + #region Messages + public sealed partial class Bar : pb::IMessage { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new Bar()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public static pbr::MessageDescriptor Descriptor { + get { return global::UnitTest.Issues.TestProtos.UnittestIssue6936CReflection.Descriptor.MessageTypes[0]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public Bar() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public Bar(Bar other) : this() { + foo_ = other.foo_ != null ? other.foo_.Clone() : null; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public Bar Clone() { + return new Bar(this); + } + + /// Field number for the "foo" field. + public const int FooFieldNumber = 1; + private global::UnitTest.Issues.TestProtos.Foo foo_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public global::UnitTest.Issues.TestProtos.Foo Foo { + get { return foo_; } + set { + foo_ = value; + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override bool Equals(object other) { + return Equals(other as Bar); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public bool Equals(Bar other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (!object.Equals(Foo, other.Foo)) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override int GetHashCode() { + int hash = 1; + if (foo_ != null) hash ^= Foo.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void WriteTo(pb::CodedOutputStream output) { + if (foo_ != null) { + output.WriteRawTag(10); + output.WriteMessage(Foo); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public int CalculateSize() { + int size = 0; + if (foo_ != null) { + size += 1 + pb::CodedOutputStream.ComputeMessageSize(Foo); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void MergeFrom(Bar other) { + if (other == null) { + return; + } + if (other.foo_ != null) { + if (foo_ == null) { + Foo = new global::UnitTest.Issues.TestProtos.Foo(); + } + Foo.MergeFrom(other.Foo); + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void MergeFrom(pb::CodedInputStream input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 10: { + if (foo_ == null) { + Foo = new global::UnitTest.Issues.TestProtos.Foo(); + } + input.ReadMessage(Foo); + break; + } + } + } + } + + } + + #endregion + +} + +#endregion Designer generated code diff --git a/csharp/src/Google.Protobuf.Test/Reflection/CustomOptionsTest.cs b/csharp/src/Google.Protobuf.Test/Reflection/CustomOptionsTest.cs index d65a6f2023..bfee5f5d43 100644 --- a/csharp/src/Google.Protobuf.Test/Reflection/CustomOptionsTest.cs +++ b/csharp/src/Google.Protobuf.Test/Reflection/CustomOptionsTest.cs @@ -193,6 +193,16 @@ namespace Google.Protobuf.Test.Reflection Assert.NotNull(TestAllTypes.Descriptor.Oneofs[0].CustomOptions); } + [Test] + public void MultipleImportOfSameFileWithExtension() + { + var descriptor = UnittestIssue6936CReflection.Descriptor; + var foo = Foo.Descriptor; + var bar = Bar.Descriptor; + AssertOption("foo", foo.CustomOptions.TryGetString, UnittestIssue6936AExtensions.Opt, foo.GetOption); + AssertOption("bar", bar.CustomOptions.TryGetString, UnittestIssue6936AExtensions.Opt, bar.GetOption); + } + private void AssertOption(T expected, OptionFetcher fetcher, Extension extension, Func, T> descriptorOptionFetcher) where D : IExtendableMessage { T customOptionsValue; diff --git a/csharp/src/Google.Protobuf.Test/testprotos.pb b/csharp/src/Google.Protobuf.Test/testprotos.pb index 818b227b37a8c1e3622a99542859ec46aa5d1b84..d00db7fdeb7639fc7be4590acb156234abafd3f3 100644 GIT binary patch delta 1238 zcmbW0zi-n(6vyv;mpFZK9BM^0jf6NN0t8i&NTeV-c0gibsEQaXr*RZltuL}Kg#nbR zR4~Dk0Rc7ns~ z{VeByF(`BcUnwtAn|>5^z51P6ebepVf44*n{mbvK@YNcy*SDU~lJ(;`6ZFQyI+GpM z|9&KTUkpe(oDiEr3aI5ULL99OhZ!WVRmx$U+=c}(Ql*jnM!{gjVL)zHKU1;M_9M~q zcRYdLZsawE8;On^Dj{3K6*@PelM^i&VhAQEbcP@vdZO2q!F4494{s@{Zwp_oh)15- z?&6KMyXWbBf43v?J$%RyL_?~!08#WFD=&zA8ALN>H!7V_Dp?e6&^#k#D-HpysYy+T zTe^Cz&%~NCdHRmXt#>`!Si!EE&vFJ8zJyF(f{Ll$=7+@-0H*#Nla@q1BiFPN_GG=|3nH z=k(;%d9;#iE7Pg8wvrPQ?dHpnfh20|Qm3TX6Kj9Zw{}Zg+xwffbI-<2QU|u?hR!w6 zrspY6M!K4-oK8HdAenU?Fv8Pt!?jDtM&SXr^#*ozxH88ptlGv$;xil>+s=-;z8I9*r$L!N9sB_@+9^l? delta 34 qcmX>yQ{+~kNJ9%_3)2?ndz#D#bhxG`Sg}a8r|L6rPt|9MzY754M+_(c diff --git a/csharp/src/Google.Protobuf/ExtensionRegistry.cs b/csharp/src/Google.Protobuf/ExtensionRegistry.cs index 2318e449e6..e631ba353d 100644 --- a/csharp/src/Google.Protobuf/ExtensionRegistry.cs +++ b/csharp/src/Google.Protobuf/ExtensionRegistry.cs @@ -42,6 +42,19 @@ namespace Google.Protobuf /// public sealed class ExtensionRegistry : ICollection, IDeepCloneable { + internal sealed class ExtensionComparer : IEqualityComparer + { + public bool Equals(Extension a, Extension b) + { + return new ObjectIntPair(a.TargetType, a.FieldNumber).Equals(new ObjectIntPair(b.TargetType, b.FieldNumber)); + } + public int GetHashCode(Extension a) + { + return new ObjectIntPair(a.TargetType, a.FieldNumber).GetHashCode(); + } + + internal static ExtensionComparer Instance = new ExtensionComparer(); + } private IDictionary, Extension> extensions; /// diff --git a/csharp/src/Google.Protobuf/Reflection/FileDescriptor.cs b/csharp/src/Google.Protobuf/Reflection/FileDescriptor.cs index 388a40b4bb..56c0caacfd 100644 --- a/csharp/src/Google.Protobuf/Reflection/FileDescriptor.cs +++ b/csharp/src/Google.Protobuf/Reflection/FileDescriptor.cs @@ -422,7 +422,8 @@ namespace Google.Protobuf.Reflection GeneratedClrTypeInfo generatedCodeInfo) { ExtensionRegistry registry = new ExtensionRegistry(); - AddAllExtensions(dependencies, generatedCodeInfo, registry); + registry.AddRange(GetAllExtensions(dependencies, generatedCodeInfo)); + FileDescriptorProto proto; try { @@ -445,9 +446,9 @@ namespace Google.Protobuf.Reflection } } - private static void AddAllExtensions(FileDescriptor[] dependencies, GeneratedClrTypeInfo generatedInfo, ExtensionRegistry registry) + private static IEnumerable GetAllExtensions(FileDescriptor[] dependencies, GeneratedClrTypeInfo generatedInfo) { - registry.AddRange(dependencies.SelectMany(GetAllDependedExtensions).Concat(GetAllGeneratedExtensions(generatedInfo)).ToArray()); + return dependencies.SelectMany(GetAllDependedExtensions).Distinct(ExtensionRegistry.ExtensionComparer.Instance).Concat(GetAllGeneratedExtensions(generatedInfo)); } private static IEnumerable GetAllGeneratedExtensions(GeneratedClrTypeInfo generated) From 537c5aa9e043865073d4e97c0d5c56f07edd5f40 Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Tue, 11 Feb 2020 15:44:40 -0800 Subject: [PATCH 15/27] Aggregate Metadata Files (#7155) (#7194) Instead of calling initOnce of dependencies, initialize metadata of dependencies in the same file. Needs to pass aggregate_metadata option to protoc to trigger, e.g.: --php_out=aggregate_metadata=foo#bar:generated_dir For each input file, transitive dependencies (including itself), whose package name has the prefix of foo or bar, will be aggregated, in which their metadata string will be aggregated in the same internalAddGeneratedFile call. For other dependencies, initOnce is called as before. This feature is EXPERIMENTAL. DO NOT USE!!! --- php/composer.json | 3 +- php/ext/google/protobuf/def.c | 136 +++---- .../Protobuf/Internal/DescriptorPool.php | 36 +- php/src/Google/Protobuf/Internal/Message.php | 1 + .../proto/test_import_descriptor_proto.proto | 2 + php/tests/well_known_test.php | 1 + .../protobuf/compiler/php/php_generator.cc | 339 ++++++++++++++---- .../protobuf/compiler/php/php_generator.h | 14 + 8 files changed, 387 insertions(+), 145 deletions(-) diff --git a/php/composer.json b/php/composer.json index b47065b78b..b618ea1507 100644 --- a/php/composer.json +++ b/php/composer.json @@ -23,6 +23,7 @@ } }, "scripts": { - "test": "(cd tests && rm -rf generated && mkdir -p generated && ../../src/protoc --php_out=generated -I../../src -I. proto/empty/echo.proto proto/test.proto proto/test_include.proto proto/test_no_namespace.proto proto/test_prefix.proto proto/test_php_namespace.proto proto/test_empty_php_namespace.proto proto/test_reserved_enum_lower.proto proto/test_reserved_enum_upper.proto proto/test_reserved_enum_value_lower.proto proto/test_reserved_enum_value_upper.proto proto/test_reserved_message_lower.proto proto/test_reserved_message_upper.proto proto/test_service.proto proto/test_service_namespace.proto proto/test_wrapper_type_setters.proto proto/test_descriptors.proto) && (cd ../src && ./protoc --php_out=../php/tests/generated -I../php/tests -I. ../php/tests/proto/test_import_descriptor_proto.proto) && vendor/bin/phpunit" + "test": "(cd tests && rm -rf generated && mkdir -p generated && ../../src/protoc --php_out=generated -I../../src -I. proto/empty/echo.proto proto/test.proto proto/test_include.proto proto/test_no_namespace.proto proto/test_prefix.proto proto/test_php_namespace.proto proto/test_empty_php_namespace.proto proto/test_reserved_enum_lower.proto proto/test_reserved_enum_upper.proto proto/test_reserved_enum_value_lower.proto proto/test_reserved_enum_value_upper.proto proto/test_reserved_message_lower.proto proto/test_reserved_message_upper.proto proto/test_service.proto proto/test_service_namespace.proto proto/test_wrapper_type_setters.proto proto/test_descriptors.proto) && (cd ../src && ./protoc --php_out=../php/tests/generated -I../php/tests -I. ../php/tests/proto/test_import_descriptor_proto.proto) && vendor/bin/phpunit", + "aggregate_metadata_test": "(cd tests && rm -rf generated && mkdir -p generated && ../../src/protoc --php_out=aggregate_metadata=foo#bar:generated -I../../src -I. proto/test.proto proto/test_include.proto && ../../src/protoc --php_out=generated -I../../src -I. proto/empty/echo.proto proto/test_no_namespace.proto proto/test_empty_php_namespace.proto proto/test_prefix.proto proto/test_php_namespace.proto proto/test_reserved_enum_lower.proto proto/test_reserved_enum_upper.proto proto/test_reserved_enum_value_lower.proto proto/test_reserved_enum_value_upper.proto proto/test_reserved_message_lower.proto proto/test_reserved_message_upper.proto proto/test_service.proto proto/test_service_namespace.proto proto/test_wrapper_type_setters.proto proto/test_descriptors.proto) && (cd ../src && ./protoc --php_out=aggregate_metadata=foo:../php/tests/generated -I../php/tests -I. ../php/tests/proto/test_import_descriptor_proto.proto) && vendor/bin/phpunit" } } diff --git a/php/ext/google/protobuf/def.c b/php/ext/google/protobuf/def.c index 7f5004ced1..5edcb03516 100644 --- a/php/ext/google/protobuf/def.c +++ b/php/ext/google/protobuf/def.c @@ -891,74 +891,15 @@ bool depends_on_descriptor(const google_protobuf_FileDescriptorProto* file) { return false; } -const upb_filedef *parse_and_add_descriptor(const char *data, - PHP_PROTO_SIZE data_len, - InternalDescriptorPoolImpl *pool, - upb_arena *arena) { - size_t n; - google_protobuf_FileDescriptorSet *set; - const google_protobuf_FileDescriptorProto* const* files; - const upb_filedef* file; - upb_status status; - - set = google_protobuf_FileDescriptorSet_parse( - data, data_len, arena); - - if (!set) { - zend_error(E_ERROR, "Failed to parse binary descriptor\n"); - return NULL; - } - - files = google_protobuf_FileDescriptorSet_file(set, &n); - - if (n != 1) { - zend_error(E_ERROR, "Serialized descriptors should have exactly one file"); - return NULL; - } - - // Check whether file has already been added. - upb_strview name = google_protobuf_FileDescriptorProto_name(files[0]); - // TODO(teboring): Needs another look up method which takes data and length. - file = upb_symtab_lookupfile2(pool->symtab, name.data, name.size); - if (file != NULL) { - return NULL; - } - - // The PHP code generator currently special-cases descriptor.proto. It - // doesn't add it as a dependency even if the proto file actually does - // depend on it. - if (depends_on_descriptor(files[0]) && - upb_symtab_lookupfile(pool->symtab, "google/protobuf/descriptor.proto") == - NULL) { - if (!parse_and_add_descriptor((char *)descriptor_proto, - descriptor_proto_len, pool, arena)) { - return NULL; - } - } - - upb_status_clear(&status); - file = upb_symtab_addfile(pool->symtab, files[0], &status); - check_upb_status(&status, "Unable to load descriptor"); - return file; -} - -void internal_add_generated_file(const char *data, PHP_PROTO_SIZE data_len, - InternalDescriptorPoolImpl *pool, - bool use_nested_submsg TSRMLS_DC) { - int i; - upb_arena *arena; - const upb_filedef* file; - - arena = upb_arena_new(); - file = parse_and_add_descriptor(data, data_len, pool, arena); - upb_arena_free(arena); - if (!file) return; - +static void internal_add_single_generated_file( + const upb_filedef* file, + InternalDescriptorPoolImpl* pool, + bool use_nested_submsg TSRMLS_DC) { + size_t i; // For each enum/message, we need its PHP class, upb descriptor and its PHP // wrapper. These information are needed later for encoding, decoding and type // checking. However, sometimes we just have one of them. In order to find // them quickly, here, we store the mapping for them. - for (i = 0; i < upb_filedef_msgcount(file); i++) { const upb_msgdef *msgdef = upb_filedef_msg(file, i); CREATE_HASHTABLE_VALUE(desc, desc_php, Descriptor, descriptor_type); @@ -1000,6 +941,73 @@ void internal_add_generated_file(const char *data, PHP_PROTO_SIZE data_len, } } +const bool parse_and_add_descriptor(const char *data, + PHP_PROTO_SIZE data_len, + InternalDescriptorPoolImpl *pool, + upb_arena *arena, + bool use_nested_submsg TSRMLS_DC) { + size_t i, n; + google_protobuf_FileDescriptorSet *set; + const google_protobuf_FileDescriptorProto* const* files; + const upb_filedef* file; + upb_status status; + + set = google_protobuf_FileDescriptorSet_parse( + data, data_len, arena); + + if (!set) { + zend_error(E_ERROR, "Failed to parse binary descriptor\n"); + return false; + } + + files = google_protobuf_FileDescriptorSet_file(set, &n); + + for (i = 0; i < n; i++) { + // Check whether file has already been added. + upb_strview name = google_protobuf_FileDescriptorProto_name(files[i]); + // TODO(teboring): Needs another look up method which takes data and length. + file = upb_symtab_lookupfile2(pool->symtab, name.data, name.size); + if (file != NULL) { + continue; + } + + // The PHP code generator currently special-cases descriptor.proto. It + // doesn't add it as a dependency even if the proto file actually does + // depend on it. + if (depends_on_descriptor(files[i]) && + upb_symtab_lookupfile( + pool->symtab, "google/protobuf/descriptor.proto") == + NULL) { + if (!parse_and_add_descriptor((char *)descriptor_proto, + descriptor_proto_len, pool, arena, + use_nested_submsg TSRMLS_CC)) { + return false; + } + } + + upb_status_clear(&status); + file = upb_symtab_addfile(pool->symtab, files[i], &status); + check_upb_status(&status, "Unable to load descriptor"); + + internal_add_single_generated_file(file, pool, use_nested_submsg TSRMLS_CC); + } + + return true; +} + +void internal_add_generated_file(const char *data, PHP_PROTO_SIZE data_len, + InternalDescriptorPoolImpl *pool, + bool use_nested_submsg TSRMLS_DC) { + int i; + upb_arena *arena; + + arena = upb_arena_new(); + parse_and_add_descriptor(data, data_len, pool, arena, + use_nested_submsg TSRMLS_CC); + upb_arena_free(arena); + return; +} + PHP_METHOD(InternalDescriptorPool, internalAddGeneratedFile) { char *data = NULL; PHP_PROTO_SIZE data_len; diff --git a/php/src/Google/Protobuf/Internal/DescriptorPool.php b/php/src/Google/Protobuf/Internal/DescriptorPool.php index 419bbf4dca..f96b0fb766 100644 --- a/php/src/Google/Protobuf/Internal/DescriptorPool.php +++ b/php/src/Google/Protobuf/Internal/DescriptorPool.php @@ -59,22 +59,25 @@ class DescriptorPool { $files = new FileDescriptorSet(); $files->mergeFromString($data); - $file = FileDescriptor::buildFromProto($files->getFile()[0]); - foreach ($file->getMessageType() as $desc) { - $this->addDescriptor($desc); - } - unset($desc); + foreach($files->getFile() as $file_proto) { + $file = FileDescriptor::buildFromProto($file_proto); - foreach ($file->getEnumType() as $desc) { - $this->addEnumDescriptor($desc); - } - unset($desc); + foreach ($file->getMessageType() as $desc) { + $this->addDescriptor($desc); + } + unset($desc); - foreach ($file->getMessageType() as $desc) { - $this->crossLink($desc); + foreach ($file->getEnumType() as $desc) { + $this->addEnumDescriptor($desc); + } + unset($desc); + + foreach ($file->getMessageType() as $desc) { + $this->crossLink($desc); + } + unset($desc); } - unset($desc); } public function addMessage($name, $klass) @@ -149,8 +152,13 @@ class DescriptorPool switch ($field->getType()) { case GPBType::MESSAGE: $proto = $field->getMessageType(); - $field->setMessageType( - $this->getDescriptorByProtoName($proto)); + $subdesc = $this->getDescriptorByProtoName($proto); + if (is_null($subdesc)) { + trigger_error( + 'proto not added: ' . $proto + . " for " . $desc->getFullName(), E_ERROR); + } + $field->setMessageType($subdesc); break; case GPBType::ENUM: $proto = $field->getEnumType(); diff --git a/php/src/Google/Protobuf/Internal/Message.php b/php/src/Google/Protobuf/Internal/Message.php index 91313336cf..8a69a13ff8 100644 --- a/php/src/Google/Protobuf/Internal/Message.php +++ b/php/src/Google/Protobuf/Internal/Message.php @@ -94,6 +94,7 @@ class Message $this->desc = $pool->getDescriptorByClassName(get_class($this)); if (is_null($this->desc)) { user_error(get_class($this) . " is not found in descriptor pool."); + return; } foreach ($this->desc->getField() as $field) { $setter = $field->getSetter(); diff --git a/php/tests/proto/test_import_descriptor_proto.proto b/php/tests/proto/test_import_descriptor_proto.proto index 2a19940dec..b06160153f 100644 --- a/php/tests/proto/test_import_descriptor_proto.proto +++ b/php/tests/proto/test_import_descriptor_proto.proto @@ -1,5 +1,7 @@ syntax = "proto3"; +package foo; + import "google/protobuf/descriptor.proto"; message TestImportDescriptorProto { diff --git a/php/tests/well_known_test.php b/php/tests/well_known_test.php index a16e070a45..a148fa4a5b 100644 --- a/php/tests/well_known_test.php +++ b/php/tests/well_known_test.php @@ -4,6 +4,7 @@ require_once('test_base.php'); require_once('test_util.php'); use Foo\TestMessage; +use Foo\TestImportDescriptorProto; use Google\Protobuf\Any; use Google\Protobuf\Api; use Google\Protobuf\BoolValue; diff --git a/src/google/protobuf/compiler/php/php_generator.cc b/src/google/protobuf/compiler/php/php_generator.cc index 55b3de2316..41263545bf 100644 --- a/src/google/protobuf/compiler/php/php_generator.cc +++ b/src/google/protobuf/compiler/php/php_generator.cc @@ -37,6 +37,16 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include @@ -91,6 +101,9 @@ std::string UnderscoresToCamelCase(const string& name, bool cap_first_letter); std::string BinaryToHex(const string& binary); void Indent(io::Printer* printer); void Outdent(io::Printer* printer); +void GenerateAddFilesToPool(const FileDescriptor* file, + const std::set& aggregate_metadata_prefixes, + io::Printer* printer); void GenerateMessageDocComment(io::Printer* printer, const Descriptor* message, int is_descriptor); void GenerateMessageConstructorDocComment(io::Printer* printer, @@ -111,7 +124,6 @@ void GenerateServiceDocComment(io::Printer* printer, void GenerateServiceMethodDocComment(io::Printer* printer, const MethodDescriptor* method); - std::string ReservedNamePrefix(const string& classname, const FileDescriptor* file) { bool is_reserved = false; @@ -924,13 +936,20 @@ void GenerateMessageToPool(const string& name_prefix, const Descriptor* message, } } -void GenerateAddFileToPool(const FileDescriptor* file, bool is_descriptor, - io::Printer* printer) { - printer->Print( - "public static $is_initialized = false;\n\n" - "public static function initOnce() {\n"); - Indent(printer); +void GenerateAddFileToPool( + const FileDescriptor* file, + bool is_descriptor, + bool aggregate_metadata, + const std::set& aggregate_metadata_prefixes, + io::Printer* printer) { + printer->Print( + "public static $is_initialized = false;\n\n" + "public static function initOnce() {\n"); + Indent(printer); + if (aggregate_metadata) { + GenerateAddFilesToPool(file, aggregate_metadata_prefixes, printer); + } else { printer->Print( "$pool = \\Google\\Protobuf\\Internal\\" "DescriptorPool::getGeneratedPool();\n\n" @@ -938,79 +957,210 @@ void GenerateAddFileToPool(const FileDescriptor* file, bool is_descriptor, " return;\n" "}\n"); - if (is_descriptor) { - for (int i = 0; i < file->message_type_count(); i++) { - GenerateMessageToPool("", file->message_type(i), printer); - } - for (int i = 0; i < file->enum_type_count(); i++) { - GenerateEnumToPool(file->enum_type(i), printer); - } + if (is_descriptor) { + for (int i = 0; i < file->message_type_count(); i++) { + GenerateMessageToPool("", file->message_type(i), printer); + } + for (int i = 0; i < file->enum_type_count(); i++) { + GenerateEnumToPool(file->enum_type(i), printer); + } + + printer->Print( + "$pool->finish();\n"); + } else { + for (int i = 0; i < file->dependency_count(); i++) { + const std::string& name = file->dependency(i)->name(); + // Currently, descriptor.proto is not ready for external usage. Skip to + // import it for now, so that its dependencies can still work as long as + // they don't use protos defined in descriptor.proto. + if (name == kDescriptorFile) { + continue; + } + std::string dependency_filename = + GeneratedMetadataFileName(file->dependency(i), is_descriptor); + printer->Print( + "\\^name^::initOnce();\n", + "name", FilenameToClassname(dependency_filename)); + } + + // Add messages and enums to descriptor pool. + FileDescriptorSet files; + FileDescriptorProto* file_proto = files.add_file(); + file->CopyTo(file_proto); + + // Filter out descriptor.proto as it cannot be depended on for now. + RepeatedPtrField* dependency = file_proto->mutable_dependency(); + for (RepeatedPtrField::iterator it = dependency->begin(); + it != dependency->end(); ++it) { + if (*it != kDescriptorFile) { + dependency->erase(it); + break; + } + } + + // Filter out all extensions, since we do not support extension yet. + file_proto->clear_extension(); + RepeatedPtrField* message_type = + file_proto->mutable_message_type(); + for (RepeatedPtrField::iterator it = message_type->begin(); + it != message_type->end(); ++it) { + it->clear_extension(); + } + + string files_data; + files.SerializeToString(&files_data); + printer->Print("$pool->internalAddGeneratedFile(hex2bin(\n"); + Indent(printer); + + printer->Print( + "\"^data^\"\n", + "data", BinaryToHex(files_data)); + + Outdent(printer); + printer->Print( + "), true);\n\n"); + } printer->Print( - "$pool->finish();\n"); + "static::$is_initialized = true;\n"); + } + + Outdent(printer); + printer->Print("}\n"); +} + +static void AnalyzeDependencyForFile( + const FileDescriptor* file, + std::set* nodes_without_dependency, + std::map>* deps, + std::map* dependency_count) { + int count = file->dependency_count(); + for (int i = 0; i < file->dependency_count(); i++) { + const FileDescriptor* dependency = file->dependency(i); + if (dependency->name() == kDescriptorFile) { + count--; + break; + } + } + + if (count == 0) { + nodes_without_dependency->insert(file); } else { + (*dependency_count)[file] = count; for (int i = 0; i < file->dependency_count(); i++) { - const std::string& name = file->dependency(i)->name(); - // Currently, descriptor.proto is not ready for external usage. Skip to - // import it for now, so that its dependencies can still work as long as - // they don't use protos defined in descriptor.proto. - if (name == kDescriptorFile) { + const FileDescriptor* dependency = file->dependency(i); + if (dependency->name() == kDescriptorFile) { continue; } - std::string dependency_filename = - GeneratedMetadataFileName(file->dependency(i), is_descriptor); - printer->Print( - "\\^name^::initOnce();\n", - "name", FilenameToClassname(dependency_filename)); + if (deps->find(dependency) == deps->end()) { + (*deps)[dependency] = std::set(); + } + (*deps)[dependency].insert(file); + AnalyzeDependencyForFile( + dependency, nodes_without_dependency, deps, dependency_count); } + } +} - // Add messages and enums to descriptor pool. - FileDescriptorSet files; - FileDescriptorProto* file_proto = files.add_file(); - file->CopyTo(file_proto); - - // Filter out descriptor.proto as it cannot be depended on for now. - RepeatedPtrField* dependency = file_proto->mutable_dependency(); - for (RepeatedPtrField::iterator it = dependency->begin(); - it != dependency->end(); ++it) { - if (*it != kDescriptorFile) { - dependency->erase(it); +static bool NeedsUnwrapping( + const FileDescriptor* file, + const std::set& aggregate_metadata_prefixes) { + bool has_aggregate_metadata_prefix = false; + if (aggregate_metadata_prefixes.empty()) { + has_aggregate_metadata_prefix = true; + } else { + for (const auto& prefix : aggregate_metadata_prefixes) { + if (HasPrefixString(file->package(), prefix)) { + has_aggregate_metadata_prefix = true; break; } } + } + + return has_aggregate_metadata_prefix; +} - // Filter out all extensions, since we do not support extension yet. - file_proto->clear_extension(); - RepeatedPtrField* message_type = - file_proto->mutable_message_type(); - for (RepeatedPtrField::iterator it = message_type->begin(); - it != message_type->end(); ++it) { - it->clear_extension(); +void GenerateAddFilesToPool( + const FileDescriptor* file, + const std::set& aggregate_metadata_prefixes, + io::Printer* printer) { + printer->Print( + "$pool = \\Google\\Protobuf\\Internal\\" + "DescriptorPool::getGeneratedPool();\n" + "if (static::$is_initialized == true) {\n" + " return;\n" + "}\n"); + + // Sort files according to dependency + std::map> deps; + std::map dependency_count; + std::set nodes_without_dependency; + FileDescriptorSet sorted_file_set; + + AnalyzeDependencyForFile( + file, &nodes_without_dependency, &deps, &dependency_count); + + while (!nodes_without_dependency.empty()) { + auto file = *nodes_without_dependency.begin(); + nodes_without_dependency.erase(file); + for (auto dependent : deps[file]) { + if (dependency_count[dependent] == 1) { + dependency_count.erase(dependent); + nodes_without_dependency.insert(dependent); + } else { + dependency_count[dependent] -= 1; + } } - string files_data; - files.SerializeToString(&files_data); + bool needs_aggregate = NeedsUnwrapping(file, aggregate_metadata_prefixes); - printer->Print("$pool->internalAddGeneratedFile(hex2bin(\n"); - Indent(printer); + if (needs_aggregate) { + auto file_proto = sorted_file_set.add_file(); + file->CopyTo(file_proto); + + // Filter out descriptor.proto as it cannot be depended on for now. + RepeatedPtrField* dependency = file_proto->mutable_dependency(); + for (RepeatedPtrField::iterator it = dependency->begin(); + it != dependency->end(); ++it) { + if (*it != kDescriptorFile) { + dependency->erase(it); + break; + } + } - // Only write 30 bytes per line. - static const int kBytesPerLine = 30; - for (int i = 0; i < files_data.size(); i += kBytesPerLine) { + // Filter out all extensions, since we do not support extension yet. + file_proto->clear_extension(); + RepeatedPtrField* message_type = + file_proto->mutable_message_type(); + for (RepeatedPtrField::iterator it = message_type->begin(); + it != message_type->end(); ++it) { + it->clear_extension(); + } + } else { + std::string dependency_filename = + GeneratedMetadataFileName(file, false); printer->Print( - "\"^data^\"^dot^\n", - "data", BinaryToHex(files_data.substr(i, kBytesPerLine)), - "dot", i + kBytesPerLine < files_data.size() ? " ." : ""); + "\\^name^::initOnce();\n", + "name", FilenameToClassname(dependency_filename)); } - - Outdent(printer); - printer->Print( - "), true);\n\n"); } + + string files_data; + sorted_file_set.SerializeToString(&files_data); + + printer->Print("$pool->internalAddGeneratedFile(hex2bin(\n"); + Indent(printer); + printer->Print( - "static::$is_initialized = true;\n"); + "\"^data^\"\n", + "data", BinaryToHex(files_data)); + Outdent(printer); - printer->Print("}\n"); + printer->Print( + "), true);\n"); + + printer->Print( + "static::$is_initialized = true;\n"); } void GenerateUseDeclaration(bool is_descriptor, io::Printer* printer) { @@ -1051,6 +1201,8 @@ std::string FilenameToClassname(const string& filename) { void GenerateMetadataFile(const FileDescriptor* file, bool is_descriptor, + bool aggregate_metadata, + const std::set& aggregate_metadata_prefixes, GeneratorContext* generator_context) { std::string filename = GeneratedMetadataFileName(file, is_descriptor); std::unique_ptr output( @@ -1079,7 +1231,8 @@ void GenerateMetadataFile(const FileDescriptor* file, } Indent(&printer); - GenerateAddFileToPool(file, is_descriptor, &printer); + GenerateAddFileToPool(file, is_descriptor, aggregate_metadata, + aggregate_metadata_prefixes, &printer); Outdent(&printer); printer.Print("}\n\n"); @@ -1229,6 +1382,7 @@ void GenerateEnumFile(const FileDescriptor* file, const EnumDescriptor* en, void GenerateMessageFile(const FileDescriptor* file, const Descriptor* message, bool is_descriptor, + bool aggregate_metadata, GeneratorContext* generator_context) { // Don't generate MapEntry messages -- we use the PHP extension's native // support for map fields instead. @@ -1285,10 +1439,12 @@ void GenerateMessageFile(const FileDescriptor* file, const Descriptor* message, GeneratedMetadataFileName(file, is_descriptor); std::string metadata_fullname = FilenameToClassname(metadata_filename); printer.Print( - "\\^fullname^::initOnce();\n" - "parent::__construct($data);\n", + "\\^fullname^::initOnce();\n", "fullname", metadata_fullname); + printer.Print( + "parent::__construct($data);\n"); + Outdent(&printer); printer.Print("}\n\n"); @@ -1328,6 +1484,7 @@ void GenerateMessageFile(const FileDescriptor* file, const Descriptor* message, // Nested messages and enums. for (int i = 0; i < message->nested_type_count(); i++) { GenerateMessageFile(file, message->nested_type(i), is_descriptor, + aggregate_metadata, generator_context); } for (int i = 0; i < message->enum_type_count(); i++) { @@ -1384,10 +1541,15 @@ void GenerateServiceFile(const FileDescriptor* file, } void GenerateFile(const FileDescriptor* file, bool is_descriptor, + bool aggregate_metadata, + const std::set& aggregate_metadata_prefixes, GeneratorContext* generator_context) { - GenerateMetadataFile(file, is_descriptor, generator_context); + GenerateMetadataFile(file, is_descriptor, aggregate_metadata, + aggregate_metadata_prefixes, generator_context); + for (int i = 0; i < file->message_type_count(); i++) { GenerateMessageFile(file, file->message_type(i), is_descriptor, + aggregate_metadata, generator_context); } for (int i = 0; i < file->enum_type_count(); i++) { @@ -1397,7 +1559,7 @@ void GenerateFile(const FileDescriptor* file, bool is_descriptor, if (file->options().php_generic_services()) { for (int i = 0; i < file->service_count(); i++) { GenerateServiceFile(file, file->service(i), is_descriptor, - generator_context); + generator_context); } } } @@ -1653,8 +1815,17 @@ void GenerateServiceMethodDocComment(io::Printer* printer, bool Generator::Generate(const FileDescriptor* file, const string& parameter, GeneratorContext* generator_context, string* error) const { - bool is_descriptor = parameter == "internal"; + return Generate(file, false, false, std::set(), + generator_context, error); +} +bool Generator::Generate( + const FileDescriptor* file, + bool is_descriptor, + bool aggregate_metadata, + const std::set& aggregate_metadata_prefixes, + GeneratorContext* generator_context, + string* error) const { if (is_descriptor && file->name() != kDescriptorFile) { *error = "Can only generate PHP code for google/protobuf/descriptor.proto.\n"; @@ -1668,8 +1839,44 @@ bool Generator::Generate(const FileDescriptor* file, const string& parameter, return false; } - GenerateFile(file, is_descriptor, generator_context); + GenerateFile(file, is_descriptor, aggregate_metadata, + aggregate_metadata_prefixes, generator_context); + + return true; +} +bool Generator::GenerateAll(const std::vector& files, + const std::string& parameter, + GeneratorContext* generator_context, + std::string* error) const { + bool is_descriptor = false; + bool aggregate_metadata = false; + std::set aggregate_metadata_prefixes; + + for (const auto& option : Split(parameter, ",")) { + const auto option_pair = Split(option, "="); + if (HasPrefixString(option_pair[0], "aggregate_metadata")) { + string options_string = option_pair[1]; + const auto options = Split(options_string, "#", false); + aggregate_metadata = true; + for (int i = 0; i < options.size(); i++) { + aggregate_metadata_prefixes.insert(options[i]); + GOOGLE_LOG(INFO) << options[i]; + } + } + if (option_pair[0] == "internal") { + is_descriptor = true; + } + } + + for (auto file : files) { + if (!Generate( + file, is_descriptor, aggregate_metadata, + aggregate_metadata_prefixes, + generator_context, error)) { + return false; + } + } return true; } diff --git a/src/google/protobuf/compiler/php/php_generator.h b/src/google/protobuf/compiler/php/php_generator.h index ef708ab56d..ca9d23a46a 100644 --- a/src/google/protobuf/compiler/php/php_generator.h +++ b/src/google/protobuf/compiler/php/php_generator.h @@ -44,10 +44,24 @@ namespace compiler { namespace php { class PROTOC_EXPORT Generator : public CodeGenerator { + public: virtual bool Generate( const FileDescriptor* file, const string& parameter, GeneratorContext* generator_context, + string* error) const override; + + bool GenerateAll(const std::vector& files, + const std::string& parameter, + GeneratorContext* generator_context, + std::string* error) const override; + private: + bool Generate( + const FileDescriptor* file, + bool is_descriptor, + bool aggregate_metadata, + const std::set& aggregate_metadata_prefixes, + GeneratorContext* generator_context, string* error) const; }; From 1440569231ba95796803fef914cbaf752c027df5 Mon Sep 17 00:00:00 2001 From: Rafi Kamal Date: Wed, 12 Feb 2020 11:34:51 -0800 Subject: [PATCH 16/27] Update the Xcode version number in Kokoro (#7202) (#7203) Currently all of our MacOS tests are failing with the error: "autom4te: need GNU m4 1.4 or later: /usr/bin/m4". This is likely to be happening because Kokoro has updated their Xcode version to 11.3. --- kokoro/macos/prepare_build_macos_rc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kokoro/macos/prepare_build_macos_rc b/kokoro/macos/prepare_build_macos_rc index acb2abaa98..2428750a8a 100755 --- a/kokoro/macos/prepare_build_macos_rc +++ b/kokoro/macos/prepare_build_macos_rc @@ -5,11 +5,11 @@ ## # Select Xcode version -# Remember to udpate the Xcode version when Xcode_11.0.app is not available. -# If xcode is not available, it will probaly encounter the failure for +# Remember to update the Xcode version when Xcode_11.3.app is not available. +# If xcode is not available, it will probably encounter the failure for # "autom4te: need GNU m4 1.4 or later: /usr/bin/m4" # go/kokoro/userdocs/macos/selecting_xcode.md for more information. -export DEVELOPER_DIR=/Applications/Xcode_11.0.app/Contents/Developer +export DEVELOPER_DIR=/Applications/Xcode_11.3.app/Contents/Developer ## # Select C/C++ compilers From 37fc4327eff474ebd1d5d492f872858ed1dd2f72 Mon Sep 17 00:00:00 2001 From: Rafi Kamal Date: Wed, 12 Feb 2020 12:37:19 -0800 Subject: [PATCH 17/27] Fix for wrappers with a zero value (#7195) (#7201) * Add failing tests for issues with wrapped values where the value is the default * Add test for wrapped values without a value set * Bugfix for wrapper types with default values. The previous optimizations for wrapper types had a bug that prevented wrappers from registering as "present" if the "value" field was not present on the wire. In practice the "value" field will not be serialized when it is zero, according to proto3 semantics, but due to the optimization this prevented it from creating a new object to represent the presence of the field. The fix is to ensure that if the wrapper message is present on the wire, we always initialize its value to zero. Co-authored-by: Joshua Haberman Co-authored-by: Dan Quan --- ruby/ext/google/protobuf_c/encode_decode.c | 76 ++++++++++++++------ ruby/tests/basic.rb | 80 ++++++++++++++++++++++ ruby/tests/common_tests.rb | 30 ++++++++ 3 files changed, 165 insertions(+), 21 deletions(-) diff --git a/ruby/ext/google/protobuf_c/encode_decode.c b/ruby/ext/google/protobuf_c/encode_decode.c index 713da43522..22e3efb473 100644 --- a/ruby/ext/google/protobuf_c/encode_decode.c +++ b/ruby/ext/google/protobuf_c/encode_decode.c @@ -44,6 +44,23 @@ VALUE noleak_rb_str_cat(VALUE rb_str, const char *str, long len) { return rb_str; } +bool is_wrapper(const upb_msgdef* m) { + switch (upb_msgdef_wellknowntype(m)) { + case UPB_WELLKNOWN_DOUBLEVALUE: + case UPB_WELLKNOWN_FLOATVALUE: + case UPB_WELLKNOWN_INT64VALUE: + case UPB_WELLKNOWN_UINT64VALUE: + case UPB_WELLKNOWN_INT32VALUE: + case UPB_WELLKNOWN_UINT32VALUE: + case UPB_WELLKNOWN_STRINGVALUE: + case UPB_WELLKNOWN_BYTESVALUE: + case UPB_WELLKNOWN_BOOLVALUE: + return true; + default: + return false; + } +} + // The code below also comes from upb's prototype Ruby binding, developed by // haberman@. @@ -117,19 +134,26 @@ static const void* newhandlerdata(upb_handlers* h, uint32_t ofs, int32_t hasbit) typedef struct { size_t ofs; int32_t hasbit; + upb_fieldtype_t wrapped_type; // Only for wrappers. VALUE subklass; } submsg_handlerdata_t; // Creates a handlerdata that contains offset and submessage type information. static const void *newsubmsghandlerdata(upb_handlers* h, + const upb_fielddef *f, uint32_t ofs, int32_t hasbit, VALUE subklass) { submsg_handlerdata_t *hd = ALLOC(submsg_handlerdata_t); + const upb_msgdef *subm = upb_fielddef_msgsubdef(f); hd->ofs = ofs; hd->hasbit = hasbit; hd->subklass = subklass; upb_handlers_addcleanup(h, hd, xfree); + if (is_wrapper(subm)) { + const upb_fielddef *value_f = upb_msgdef_itof(subm, 1); + hd->wrapped_type = upb_fielddef_type(value_f); + } return hd; } @@ -310,12 +334,39 @@ static void *submsg_handler(void *closure, const void *hd) { } static void* startwrapper(void* closure, const void* hd) { - char* msg = closure; const submsg_handlerdata_t* submsgdata = hd; + char* msg = closure; + VALUE* field = (VALUE*)(msg + submsgdata->ofs); set_hasbit(closure, submsgdata->hasbit); - return msg + submsgdata->ofs; + switch (submsgdata->wrapped_type) { + case UPB_TYPE_FLOAT: + case UPB_TYPE_DOUBLE: + *field = DBL2NUM(0); + break; + case UPB_TYPE_BOOL: + *field = Qfalse; + break; + case UPB_TYPE_STRING: + *field = get_frozen_string(NULL, 0, false); + break; + case UPB_TYPE_BYTES: + *field = get_frozen_string(NULL, 0, true); + break; + case UPB_TYPE_ENUM: + case UPB_TYPE_INT32: + case UPB_TYPE_INT64: + case UPB_TYPE_UINT32: + case UPB_TYPE_UINT64: + *field = INT2NUM(0); + break; + case UPB_TYPE_MESSAGE: + rb_raise(rb_eRuntimeError, + "Internal logic error with well-known types."); + } + + return field; } // Handler data for startmap/endmap handlers. @@ -522,23 +573,6 @@ static void* oneof_startwrapper(void* closure, const void* hd) { return msg + oneofdata->ofs; } -bool is_wrapper(const upb_msgdef* m) { - switch (upb_msgdef_wellknowntype(m)) { - case UPB_WELLKNOWN_DOUBLEVALUE: - case UPB_WELLKNOWN_FLOATVALUE: - case UPB_WELLKNOWN_INT64VALUE: - case UPB_WELLKNOWN_UINT64VALUE: - case UPB_WELLKNOWN_INT32VALUE: - case UPB_WELLKNOWN_UINT32VALUE: - case UPB_WELLKNOWN_STRINGVALUE: - case UPB_WELLKNOWN_BYTESVALUE: - case UPB_WELLKNOWN_BOOLVALUE: - return true; - default: - return false; - } -} - // Set up handlers for a repeated field. static void add_handlers_for_repeated_field(upb_handlers *h, const Descriptor* desc, @@ -579,7 +613,7 @@ static void add_handlers_for_repeated_field(upb_handlers *h, case UPB_TYPE_MESSAGE: { VALUE subklass = field_type_class(desc->layout, f); upb_handlerattr attr = UPB_HANDLERATTR_INIT; - attr.handler_data = newsubmsghandlerdata(h, 0, -1, subklass); + attr.handler_data = newsubmsghandlerdata(h, f, 0, -1, subklass); if (is_wrapper(upb_fielddef_msgsubdef(f))) { upb_handlers_setstartsubmsg(h, f, appendwrapper_handler, &attr); } else { @@ -708,7 +742,7 @@ static void add_handlers_for_singular_field(const Descriptor* desc, case UPB_TYPE_MESSAGE: { upb_handlerattr attr = UPB_HANDLERATTR_INIT; attr.handler_data = newsubmsghandlerdata( - h, offset, hasbit, field_type_class(desc->layout, f)); + h, f, offset, hasbit, field_type_class(desc->layout, f)); if (is_wrapper(upb_fielddef_msgsubdef(f))) { upb_handlers_setstartsubmsg(h, f, startwrapper, &attr); } else { diff --git a/ruby/tests/basic.rb b/ruby/tests/basic.rb index 1cb8d3463e..d1a66a6169 100644 --- a/ruby/tests/basic.rb +++ b/ruby/tests/basic.rb @@ -276,6 +276,86 @@ module BasicTest assert_equal m5, m end + def test_map_wrappers_with_default_values + run_asserts = ->(m) { + assert_equal 0.0, m.map_double[0].value + assert_equal 0.0, m.map_float[0].value + assert_equal 0, m.map_int32[0].value + assert_equal 0, m.map_int64[0].value + assert_equal 0, m.map_uint32[0].value + assert_equal 0, m.map_uint64[0].value + assert_equal false, m.map_bool[0].value + assert_equal '', m.map_string[0].value + assert_equal '', m.map_bytes[0].value + } + + m = proto_module::Wrapper.new( + map_double: {0 => Google::Protobuf::DoubleValue.new(value: 0.0)}, + map_float: {0 => Google::Protobuf::FloatValue.new(value: 0.0)}, + map_int32: {0 => Google::Protobuf::Int32Value.new(value: 0)}, + map_int64: {0 => Google::Protobuf::Int64Value.new(value: 0)}, + map_uint32: {0 => Google::Protobuf::UInt32Value.new(value: 0)}, + map_uint64: {0 => Google::Protobuf::UInt64Value.new(value: 0)}, + map_bool: {0 => Google::Protobuf::BoolValue.new(value: false)}, + map_string: {0 => Google::Protobuf::StringValue.new(value: '')}, + map_bytes: {0 => Google::Protobuf::BytesValue.new(value: '')}, + ) + + run_asserts.call(m) + serialized = proto_module::Wrapper::encode(m) + m2 = proto_module::Wrapper::decode(serialized) + run_asserts.call(m2) + + # Test the case where we are serializing directly from the parsed form + # (before anything lazy is materialized). + m3 = proto_module::Wrapper::decode(serialized) + serialized2 = proto_module::Wrapper::encode(m3) + m4 = proto_module::Wrapper::decode(serialized2) + run_asserts.call(m4) + + # Test that the lazy form compares equal to the expanded form. + m5 = proto_module::Wrapper::decode(serialized2) + assert_equal m5, m + end + + def test_map_wrappers_with_no_value + run_asserts = ->(m) { + assert_equal 0.0, m.map_double[0].value + assert_equal 0.0, m.map_float[0].value + assert_equal 0, m.map_int32[0].value + assert_equal 0, m.map_int64[0].value + assert_equal 0, m.map_uint32[0].value + assert_equal 0, m.map_uint64[0].value + assert_equal false, m.map_bool[0].value + assert_equal '', m.map_string[0].value + assert_equal '', m.map_bytes[0].value + } + + m = proto_module::Wrapper.new( + map_double: {0 => Google::Protobuf::DoubleValue.new()}, + map_float: {0 => Google::Protobuf::FloatValue.new()}, + map_int32: {0 => Google::Protobuf::Int32Value.new()}, + map_int64: {0 => Google::Protobuf::Int64Value.new()}, + map_uint32: {0 => Google::Protobuf::UInt32Value.new()}, + map_uint64: {0 => Google::Protobuf::UInt64Value.new()}, + map_bool: {0 => Google::Protobuf::BoolValue.new()}, + map_string: {0 => Google::Protobuf::StringValue.new()}, + map_bytes: {0 => Google::Protobuf::BytesValue.new()}, + ) + run_asserts.call(m) + + serialized = proto_module::Wrapper::encode(m) + m2 = proto_module::Wrapper::decode(serialized) + run_asserts.call(m2) + + # Test the case where we are serializing directly from the parsed form + # (before anything lazy is materialized). + m3 = proto_module::Wrapper::decode(serialized) + serialized2 = proto_module::Wrapper::encode(m3) + m4 = proto_module::Wrapper::decode(serialized2) + run_asserts.call(m4) + end + def test_concurrent_decoding o = Outer.new o.items[0] = Inner.new diff --git a/ruby/tests/common_tests.rb b/ruby/tests/common_tests.rb index ada42432eb..ee05b856dc 100644 --- a/ruby/tests/common_tests.rb +++ b/ruby/tests/common_tests.rb @@ -1265,6 +1265,36 @@ module CommonTests assert proto_module::TestMessage.new != nil end + def test_wrappers_set_to_default + run_asserts = ->(m) { + assert_equal 0.0, m.double.value + assert_equal 0.0, m.float.value + assert_equal 0, m.int32.value + assert_equal 0, m.int64.value + assert_equal 0, m.uint32.value + assert_equal 0, m.uint64.value + assert_equal false, m.bool.value + assert_equal '', m.string.value + assert_equal '', m.bytes.value + } + + m = proto_module::Wrapper.new( + double: Google::Protobuf::DoubleValue.new(value: 0.0), + float: Google::Protobuf::FloatValue.new(value: 0.0), + int32: Google::Protobuf::Int32Value.new(value: 0), + int64: Google::Protobuf::Int64Value.new(value: 0), + uint32: Google::Protobuf::UInt32Value.new(value: 0), + uint64: Google::Protobuf::UInt64Value.new(value: 0), + bool: Google::Protobuf::BoolValue.new(value: false), + string: Google::Protobuf::StringValue.new(value: ""), + bytes: Google::Protobuf::BytesValue.new(value: ''), + ) + + run_asserts.call(m) + m2 = proto_module::Wrapper.decode(m.to_proto) + run_asserts.call(m2) + end + def test_wrapper_getters run_asserts = ->(m) { assert_equal 2.0, m.double_as_value From 3c020b90a772b1177ca80e24daaeba93c3eae903 Mon Sep 17 00:00:00 2001 From: Rafi Kamal Date: Wed, 12 Feb 2020 12:51:32 -0800 Subject: [PATCH 18/27] Update CHANGES.txt for 3.11.4 release --- CHANGES.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index b047b97208..3da8143a6f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,12 @@ +2020-02-14 version 3.11.4 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) + + C# + * Fix latest ArgumentException for C# extensions (#7188) + + Ruby + * Fix wrappers with a zero value (#7195) + * Fix JSON serialization of 0/empty-valued wrapper types (#7198) + 2020-01-31 version 3.11.3 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) C++ From 906e9c689751b94f1c89da047f8f093cb7b993d2 Mon Sep 17 00:00:00 2001 From: Rafi Kamal Date: Wed, 12 Feb 2020 13:27:36 -0800 Subject: [PATCH 19/27] Fix for JSON serialization of 0/empty-valued wrapper types (#7198) (#7204) * Fixed Ruby JSON serialization of 0/empty wrapper fields. * Removed newly-passing conformance tests from the failure list. Co-authored-by: Joshua Haberman --- conformance/failure_list_ruby.txt | 18 ------------------ ruby/ext/google/protobuf_c/encode_decode.c | 5 +++-- ruby/tests/common_tests.rb | 1 + 3 files changed, 4 insertions(+), 20 deletions(-) diff --git a/conformance/failure_list_ruby.txt b/conformance/failure_list_ruby.txt index 4573c59e4e..f9533ae8ff 100644 --- a/conformance/failure_list_ruby.txt +++ b/conformance/failure_list_ruby.txt @@ -94,25 +94,7 @@ Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput Required.Proto3.JsonInput.OneofFieldDuplicate -Required.Proto3.JsonInput.OptionalBoolWrapper.JsonOutput -Required.Proto3.JsonInput.OptionalBytesWrapper.JsonOutput -Required.Proto3.JsonInput.OptionalDoubleWrapper.JsonOutput -Required.Proto3.JsonInput.OptionalFloatWrapper.JsonOutput -Required.Proto3.JsonInput.OptionalInt32Wrapper.JsonOutput -Required.Proto3.JsonInput.OptionalInt64Wrapper.JsonOutput -Required.Proto3.JsonInput.OptionalStringWrapper.JsonOutput -Required.Proto3.JsonInput.OptionalUint32Wrapper.JsonOutput -Required.Proto3.JsonInput.OptionalUint64Wrapper.JsonOutput Required.Proto3.JsonInput.RejectTopLevelNull -Required.Proto3.JsonInput.RepeatedBoolWrapper.JsonOutput -Required.Proto3.JsonInput.RepeatedBytesWrapper.JsonOutput -Required.Proto3.JsonInput.RepeatedDoubleWrapper.JsonOutput -Required.Proto3.JsonInput.RepeatedFloatWrapper.JsonOutput -Required.Proto3.JsonInput.RepeatedInt32Wrapper.JsonOutput -Required.Proto3.JsonInput.RepeatedInt64Wrapper.JsonOutput -Required.Proto3.JsonInput.RepeatedStringWrapper.JsonOutput -Required.Proto3.JsonInput.RepeatedUint32Wrapper.JsonOutput -Required.Proto3.JsonInput.RepeatedUint64Wrapper.JsonOutput Required.Proto3.JsonInput.StringFieldSurrogatePair.JsonOutput Required.Proto3.JsonInput.StringFieldSurrogatePair.ProtobufOutput Required.Proto3.ProtobufInput.DoubleFieldNormalizeQuietNan.JsonOutput diff --git a/ruby/ext/google/protobuf_c/encode_decode.c b/ruby/ext/google/protobuf_c/encode_decode.c index 22e3efb473..142fd57873 100644 --- a/ruby/ext/google/protobuf_c/encode_decode.c +++ b/ruby/ext/google/protobuf_c/encode_decode.c @@ -1466,6 +1466,7 @@ static void putmsg(VALUE msg_rb, const Descriptor* desc, MessageHeader* msg; upb_msg_field_iter i; upb_status status; + bool json_wrapper = is_wrapper(desc->msgdef) && is_json; if (is_json && upb_msgdef_wellknowntype(desc->msgdef) == UPB_WELLKNOWN_ANY) { @@ -1542,7 +1543,7 @@ static void putmsg(VALUE msg_rb, const Descriptor* desc, is_default = RSTRING_LEN(str) == 0; } - if (is_matching_oneof || emit_defaults || !is_default) { + if (is_matching_oneof || emit_defaults || !is_default || json_wrapper) { putstr(str, f, sink); } } else if (upb_fielddef_issubmsg(f)) { @@ -1562,7 +1563,7 @@ static void putmsg(VALUE msg_rb, const Descriptor* desc, } else if (upb_msgdef_syntax(desc->msgdef) == UPB_SYNTAX_PROTO3) { \ is_default = default_value == value; \ } \ - if (is_matching_oneof || emit_defaults || !is_default) { \ + if (is_matching_oneof || emit_defaults || !is_default || json_wrapper) { \ upb_sink_put##upbtype(sink, sel, value); \ } \ } break; diff --git a/ruby/tests/common_tests.rb b/ruby/tests/common_tests.rb index ee05b856dc..7414aea8ae 100644 --- a/ruby/tests/common_tests.rb +++ b/ruby/tests/common_tests.rb @@ -1293,6 +1293,7 @@ module CommonTests run_asserts.call(m) m2 = proto_module::Wrapper.decode(m.to_proto) run_asserts.call(m2) + m3 = proto_module::Wrapper.decode_json(m.to_json) end def test_wrapper_getters From c74057267d64d0b5b8a48feeab2d5b015590aca0 Mon Sep 17 00:00:00 2001 From: Rafi Kamal Date: Wed, 12 Feb 2020 14:41:16 -0800 Subject: [PATCH 20/27] Update protobuf version (#7206) --- Protobuf-C++.podspec | 2 +- Protobuf.podspec | 2 +- configure.ac | 2 +- csharp/Google.Protobuf.Tools.nuspec | 2 +- .../Google.Protobuf/Google.Protobuf.csproj | 2 +- java/bom/pom.xml | 2 +- java/core/pom.xml | 2 +- java/lite/pom.xml | 2 +- java/pom.xml | 2 +- java/util/pom.xml | 2 +- js/package.json | 2 +- php/ext/google/protobuf/package.xml | 22 +++++++++++++++---- php/ext/google/protobuf/protobuf.h | 2 +- protoc-artifacts/pom.xml | 2 +- python/google/protobuf/__init__.py | 2 +- ruby/google-protobuf.gemspec | 2 +- src/Makefile.am | 2 +- src/google/protobuf/any.pb.h | 2 +- src/google/protobuf/api.pb.h | 2 +- src/google/protobuf/compiler/plugin.pb.h | 2 +- src/google/protobuf/descriptor.pb.h | 2 +- src/google/protobuf/duration.pb.h | 2 +- src/google/protobuf/empty.pb.h | 2 +- src/google/protobuf/field_mask.pb.h | 2 +- src/google/protobuf/port_def.inc | 2 +- src/google/protobuf/source_context.pb.h | 2 +- src/google/protobuf/struct.pb.h | 2 +- src/google/protobuf/stubs/common.h | 2 +- src/google/protobuf/timestamp.pb.h | 2 +- src/google/protobuf/type.pb.h | 2 +- src/google/protobuf/wrappers.pb.h | 2 +- 31 files changed, 48 insertions(+), 34 deletions(-) diff --git a/Protobuf-C++.podspec b/Protobuf-C++.podspec index db3857765f..dcb9f4d705 100644 --- a/Protobuf-C++.podspec +++ b/Protobuf-C++.podspec @@ -1,6 +1,6 @@ Pod::Spec.new do |s| s.name = 'Protobuf-C++' - s.version = '3.11.3' + s.version = '3.11.4' s.summary = 'Protocol Buffers v3 runtime library for C++.' s.homepage = 'https://github.com/google/protobuf' s.license = '3-Clause BSD License' diff --git a/Protobuf.podspec b/Protobuf.podspec index 73d40eee25..b5436991e7 100644 --- a/Protobuf.podspec +++ b/Protobuf.podspec @@ -5,7 +5,7 @@ # dependent projects use the :git notation to refer to the library. Pod::Spec.new do |s| s.name = 'Protobuf' - s.version = '3.11.3' + s.version = '3.11.4' s.summary = 'Protocol Buffers v.3 runtime library for Objective-C.' s.homepage = 'https://github.com/protocolbuffers/protobuf' s.license = '3-Clause BSD License' diff --git a/configure.ac b/configure.ac index bf51d8b670..a86499a571 100644 --- a/configure.ac +++ b/configure.ac @@ -17,7 +17,7 @@ AC_PREREQ(2.59) # In the SVN trunk, the version should always be the next anticipated release # version with the "-pre" suffix. (We used to use "-SNAPSHOT" but this pushed # the size of one file name in the dist tarfile over the 99-char limit.) -AC_INIT([Protocol Buffers],[3.11.3],[protobuf@googlegroups.com],[protobuf]) +AC_INIT([Protocol Buffers],[3.11.4],[protobuf@googlegroups.com],[protobuf]) AM_MAINTAINER_MODE([enable]) diff --git a/csharp/Google.Protobuf.Tools.nuspec b/csharp/Google.Protobuf.Tools.nuspec index ff983da9b7..d8881869fa 100644 --- a/csharp/Google.Protobuf.Tools.nuspec +++ b/csharp/Google.Protobuf.Tools.nuspec @@ -5,7 +5,7 @@ Google Protocol Buffers tools Tools for Protocol Buffers - Google's data interchange format. See project site for more info. - 3.11.3 + 3.11.4 Google Inc. protobuf-packages https://github.com/protocolbuffers/protobuf/blob/master/LICENSE diff --git a/csharp/src/Google.Protobuf/Google.Protobuf.csproj b/csharp/src/Google.Protobuf/Google.Protobuf.csproj index 2cc7d2373f..53f45febc7 100644 --- a/csharp/src/Google.Protobuf/Google.Protobuf.csproj +++ b/csharp/src/Google.Protobuf/Google.Protobuf.csproj @@ -4,7 +4,7 @@ C# runtime library for Protocol Buffers - Google's data interchange format. Copyright 2015, Google Inc. Google Protocol Buffers - 3.11.3 + 3.11.4 6 Google Inc. netstandard1.0;netstandard2.0;net45 diff --git a/java/bom/pom.xml b/java/bom/pom.xml index d5c9f07e76..f499bd132b 100644 --- a/java/bom/pom.xml +++ b/java/bom/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-bom - 3.11.3 + 3.11.4 pom Protocol Buffers [BOM] diff --git a/java/core/pom.xml b/java/core/pom.xml index 14145f733c..79a1e714ab 100644 --- a/java/core/pom.xml +++ b/java/core/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.11.3 + 3.11.4 protobuf-java diff --git a/java/lite/pom.xml b/java/lite/pom.xml index 8504f82d69..47bc1219f0 100644 --- a/java/lite/pom.xml +++ b/java/lite/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.11.3 + 3.11.4 protobuf-javalite diff --git a/java/pom.xml b/java/pom.xml index 6eda963353..f82dd0f20d 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.11.3 + 3.11.4 pom Protocol Buffers [Parent] diff --git a/java/util/pom.xml b/java/util/pom.xml index 67b802b051..58d05d2972 100644 --- a/java/util/pom.xml +++ b/java/util/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.11.3 + 3.11.4 protobuf-java-util diff --git a/js/package.json b/js/package.json index 942beb55ca..14186bec1f 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "google-protobuf", - "version": "3.11.3", + "version": "3.11.4", "description": "Protocol Buffers for JavaScript", "main": "google-protobuf.js", "files": [ diff --git a/php/ext/google/protobuf/package.xml b/php/ext/google/protobuf/package.xml index d5daa7b078..5cb2b4648c 100644 --- a/php/ext/google/protobuf/package.xml +++ b/php/ext/google/protobuf/package.xml @@ -10,11 +10,11 @@ protobuf-opensource@google.com yes - 2020-01-28 - + 2020-02-12 + - 3.11.3 - 3.11.3 + 3.11.4 + 3.11.4 stable @@ -515,5 +515,19 @@ G A release. 3-Clause BSD License GA release. + + + 3.11.4 + 3.11.4 + + + stable + stable + + 2020-02-12 + + 3-Clause BSD License + GA release. + diff --git a/php/ext/google/protobuf/protobuf.h b/php/ext/google/protobuf/protobuf.h index f9c1c792f4..88cc4005d9 100644 --- a/php/ext/google/protobuf/protobuf.h +++ b/php/ext/google/protobuf/protobuf.h @@ -37,7 +37,7 @@ #include "upb.h" #define PHP_PROTOBUF_EXTNAME "protobuf" -#define PHP_PROTOBUF_VERSION "3.11.3" +#define PHP_PROTOBUF_VERSION "3.11.4" #define MAX_LENGTH_OF_INT64 20 #define SIZEOF_INT64 8 diff --git a/protoc-artifacts/pom.xml b/protoc-artifacts/pom.xml index 3327802099..eec7d82894 100644 --- a/protoc-artifacts/pom.xml +++ b/protoc-artifacts/pom.xml @@ -8,7 +8,7 @@ com.google.protobuf protoc - 3.11.3 + 3.11.4 pom Protobuf Compiler diff --git a/python/google/protobuf/__init__.py b/python/google/protobuf/__init__.py index c0745d0e7a..cc9a06a0ef 100755 --- a/python/google/protobuf/__init__.py +++ b/python/google/protobuf/__init__.py @@ -30,7 +30,7 @@ # Copyright 2007 Google Inc. All Rights Reserved. -__version__ = '3.11.3' +__version__ = '3.11.4' if __name__ != '__main__': try: diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index 8f51cf65a8..bf60b6ae9e 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = "google-protobuf" - s.version = "3.11.3" + s.version = "3.11.4" git_tag = "v#{s.version.to_s.sub('.rc.', '-rc')}" # Converts X.Y.Z.rc.N to vX.Y.Z-rcN, used for the git tag s.licenses = ["BSD-3-Clause"] s.summary = "Protocol Buffers" diff --git a/src/Makefile.am b/src/Makefile.am index 9d7bc4c126..7d5e779cee 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -18,7 +18,7 @@ else PTHREAD_DEF = endif -PROTOBUF_VERSION = 22:3:0 +PROTOBUF_VERSION = 22:4:0 if GCC # Turn on all warnings except for sign comparison (we ignore sign comparison diff --git a/src/google/protobuf/any.pb.h b/src/google/protobuf/any.pb.h index 87c2aedfd0..0ff9341b04 100644 --- a/src/google/protobuf/any.pb.h +++ b/src/google/protobuf/any.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/api.pb.h b/src/google/protobuf/api.pb.h index e1f32a641c..ed49df1b1f 100644 --- a/src/google/protobuf/api.pb.h +++ b/src/google/protobuf/api.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/compiler/plugin.pb.h b/src/google/protobuf/compiler/plugin.pb.h index 0040463fce..6061e3c6ba 100644 --- a/src/google/protobuf/compiler/plugin.pb.h +++ b/src/google/protobuf/compiler/plugin.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/descriptor.pb.h b/src/google/protobuf/descriptor.pb.h index 48ef8a1863..e7bae0da64 100644 --- a/src/google/protobuf/descriptor.pb.h +++ b/src/google/protobuf/descriptor.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/duration.pb.h b/src/google/protobuf/duration.pb.h index 33af624bc1..b2efc3c3f8 100644 --- a/src/google/protobuf/duration.pb.h +++ b/src/google/protobuf/duration.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/empty.pb.h b/src/google/protobuf/empty.pb.h index 2c22bd3509..7d24d8975a 100644 --- a/src/google/protobuf/empty.pb.h +++ b/src/google/protobuf/empty.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/field_mask.pb.h b/src/google/protobuf/field_mask.pb.h index 24c5c44782..af6b8a9ebf 100644 --- a/src/google/protobuf/field_mask.pb.h +++ b/src/google/protobuf/field_mask.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 375f919d70..2367e91742 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -297,7 +297,7 @@ // Shared google3/opensource definitions. ////////////////////////////////////// -#define PROTOBUF_VERSION 3011003 +#define PROTOBUF_VERSION 3011004 #define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC 3011000 #define PROTOBUF_MIN_PROTOC_VERSION 3011000 #define PROTOBUF_VERSION_SUFFIX "" diff --git a/src/google/protobuf/source_context.pb.h b/src/google/protobuf/source_context.pb.h index 77349c8a6e..2861a69db1 100644 --- a/src/google/protobuf/source_context.pb.h +++ b/src/google/protobuf/source_context.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/struct.pb.h b/src/google/protobuf/struct.pb.h index 09306bc817..46363e4b75 100644 --- a/src/google/protobuf/struct.pb.h +++ b/src/google/protobuf/struct.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/stubs/common.h b/src/google/protobuf/stubs/common.h index 453f6313f0..d7ec691892 100644 --- a/src/google/protobuf/stubs/common.h +++ b/src/google/protobuf/stubs/common.h @@ -81,7 +81,7 @@ namespace internal { // The current version, represented as a single integer to make comparison // easier: major * 10^6 + minor * 10^3 + micro -#define GOOGLE_PROTOBUF_VERSION 3011003 +#define GOOGLE_PROTOBUF_VERSION 3011004 // A suffix string for alpha, beta or rc releases. Empty for stable releases. #define GOOGLE_PROTOBUF_VERSION_SUFFIX "" diff --git a/src/google/protobuf/timestamp.pb.h b/src/google/protobuf/timestamp.pb.h index bae12ebac4..f0a4330c58 100644 --- a/src/google/protobuf/timestamp.pb.h +++ b/src/google/protobuf/timestamp.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/type.pb.h b/src/google/protobuf/type.pb.h index 074d10e668..8756c93e02 100644 --- a/src/google/protobuf/type.pb.h +++ b/src/google/protobuf/type.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/wrappers.pb.h b/src/google/protobuf/wrappers.pb.h index d62d120eb6..dd03416d58 100644 --- a/src/google/protobuf/wrappers.pb.h +++ b/src/google/protobuf/wrappers.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3011003 < PROTOBUF_MIN_PROTOC_VERSION +#if 3011004 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. From d3141015311268a2248a2443dcf7fd9b83f80464 Mon Sep 17 00:00:00 2001 From: Rafi Kamal Date: Thu, 13 Feb 2020 14:00:19 -0800 Subject: [PATCH 21/27] Remove protoc release for 32-bit Macs (#7209) * Remove protoc release for 32-bit Macs Apple has been removing the support for 32-bit Mac apps: https://support.apple.com/en-us/HT208436 Our release infrastructure no longer supports building for 32-bit architecture. * Remove protoc artifact for MacOS 32-bit --- kokoro/release/protoc/macos/build.sh | 8 -------- protoc-artifacts/pom.xml | 5 ----- 2 files changed, 13 deletions(-) diff --git a/kokoro/release/protoc/macos/build.sh b/kokoro/release/protoc/macos/build.sh index 6a4c79c7de..47c9bfa968 100644 --- a/kokoro/release/protoc/macos/build.sh +++ b/kokoro/release/protoc/macos/build.sh @@ -6,14 +6,6 @@ CXXFLAGS_COMMON="-std=c++14 -DNDEBUG -mmacosx-version-min=10.9" cd github/protobuf ./autogen.sh -mkdir build32 && cd build32 -export CXXFLAGS="$CXXFLAGS_COMMON -m32" -../configure --disable-shared -make -j4 -file src/protoc -otool -L src/protoc | grep dylib -cd .. - mkdir build64 && cd build64 export CXXFLAGS="$CXXFLAGS_COMMON -m64" ../configure --disable-shared diff --git a/protoc-artifacts/pom.xml b/protoc-artifacts/pom.xml index eec7d82894..32cbc65305 100644 --- a/protoc-artifacts/pom.xml +++ b/protoc-artifacts/pom.xml @@ -70,11 +70,6 @@ osx-x86_64 exe - - ${basedir}/target/osx/x86_32/protoc.exe - osx-x86_32 - exe - ${basedir}/target/linux/aarch_64/protoc.exe linux-aarch_64 From 0e8f69e53235260f75151d145d75e01c2ad65799 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 14 Feb 2020 19:17:06 +0100 Subject: [PATCH 22/27] enforce recursion depth checking for unknown fields (#7210) --- .../CodedInputStreamTest.cs | 63 ++++++++++++++++++- .../src/Google.Protobuf/CodedInputStream.cs | 31 ++++++++- csharp/src/Google.Protobuf/UnknownFieldSet.cs | 22 +++++-- 3 files changed, 108 insertions(+), 8 deletions(-) diff --git a/csharp/src/Google.Protobuf.Test/CodedInputStreamTest.cs b/csharp/src/Google.Protobuf.Test/CodedInputStreamTest.cs index ba65b328e8..5f360ff46e 100644 --- a/csharp/src/Google.Protobuf.Test/CodedInputStreamTest.cs +++ b/csharp/src/Google.Protobuf.Test/CodedInputStreamTest.cs @@ -33,6 +33,7 @@ using System; using System.IO; using Google.Protobuf.TestProtos; +using Proto2 = Google.Protobuf.TestProtos.Proto2; using NUnit.Framework; namespace Google.Protobuf @@ -337,6 +338,66 @@ namespace Google.Protobuf CodedInputStream input = CodedInputStream.CreateWithLimits(new MemoryStream(atRecursiveLimit.ToByteArray()), 1000000, CodedInputStream.DefaultRecursionLimit - 1); Assert.Throws(() => TestRecursiveMessage.Parser.ParseFrom(input)); } + + private static byte[] MakeMaliciousRecursionUnknownFieldsPayload(int recursionDepth) + { + // generate recursively nested groups that will be parsed as unknown fields + int unknownFieldNumber = 14; // an unused field number + MemoryStream ms = new MemoryStream(); + CodedOutputStream output = new CodedOutputStream(ms); + for (int i = 0; i < recursionDepth; i++) + { + output.WriteTag(WireFormat.MakeTag(unknownFieldNumber, WireFormat.WireType.StartGroup)); + } + for (int i = 0; i < recursionDepth; i++) + { + output.WriteTag(WireFormat.MakeTag(unknownFieldNumber, WireFormat.WireType.EndGroup)); + } + output.Flush(); + return ms.ToArray(); + } + + [Test] + public void MaliciousRecursion_UnknownFields() + { + byte[] payloadAtRecursiveLimit = MakeMaliciousRecursionUnknownFieldsPayload(CodedInputStream.DefaultRecursionLimit); + byte[] payloadBeyondRecursiveLimit = MakeMaliciousRecursionUnknownFieldsPayload(CodedInputStream.DefaultRecursionLimit + 1); + + Assert.DoesNotThrow(() => TestRecursiveMessage.Parser.ParseFrom(payloadAtRecursiveLimit)); + Assert.Throws(() => TestRecursiveMessage.Parser.ParseFrom(payloadBeyondRecursiveLimit)); + } + + [Test] + public void ReadGroup_WrongEndGroupTag() + { + int groupFieldNumber = Proto2.TestAllTypes.OptionalGroupFieldNumber; + + // write Proto2.TestAllTypes with "optional_group" set, but use wrong EndGroup closing tag + MemoryStream ms = new MemoryStream(); + CodedOutputStream output = new CodedOutputStream(ms); + output.WriteTag(WireFormat.MakeTag(groupFieldNumber, WireFormat.WireType.StartGroup)); + output.WriteGroup(new Proto2.TestAllTypes.Types.OptionalGroup { A = 12345 }); + // end group with different field number + output.WriteTag(WireFormat.MakeTag(groupFieldNumber + 1, WireFormat.WireType.EndGroup)); + output.Flush(); + var payload = ms.ToArray(); + + Assert.Throws(() => Proto2.TestAllTypes.Parser.ParseFrom(payload)); + } + + [Test] + public void ReadGroup_UnknownFields_WrongEndGroupTag() + { + MemoryStream ms = new MemoryStream(); + CodedOutputStream output = new CodedOutputStream(ms); + output.WriteTag(WireFormat.MakeTag(14, WireFormat.WireType.StartGroup)); + // end group with different field number + output.WriteTag(WireFormat.MakeTag(15, WireFormat.WireType.EndGroup)); + output.Flush(); + var payload = ms.ToArray(); + + Assert.Throws(() => TestRecursiveMessage.Parser.ParseFrom(payload)); + } [Test] public void SizeLimit() @@ -735,4 +796,4 @@ namespace Google.Protobuf } } } -} \ No newline at end of file +} diff --git a/csharp/src/Google.Protobuf/CodedInputStream.cs b/csharp/src/Google.Protobuf/CodedInputStream.cs index 44934f341b..c3ca3edef6 100644 --- a/csharp/src/Google.Protobuf/CodedInputStream.cs +++ b/csharp/src/Google.Protobuf/CodedInputStream.cs @@ -307,10 +307,17 @@ namespace Google.Protobuf throw InvalidProtocolBufferException.MoreDataAvailable(); } } - #endregion + internal void CheckLastTagWas(uint expectedTag) + { + if (lastTag != expectedTag) { + throw InvalidProtocolBufferException.InvalidEndTag(); + } + } + #endregion + #region Reading of tags etc - + /// /// Peeks at the next field tag. This is like calling , but the /// tag is not consumed. (So a subsequent call to will return the @@ -636,7 +643,27 @@ namespace Google.Protobuf throw InvalidProtocolBufferException.RecursionLimitExceeded(); } ++recursionDepth; + + uint tag = lastTag; + int fieldNumber = WireFormat.GetTagFieldNumber(tag); + builder.MergeFrom(this); + CheckLastTagWas(WireFormat.MakeTag(fieldNumber, WireFormat.WireType.EndGroup)); + --recursionDepth; + } + + /// + /// Reads an embedded group unknown field from the stream. + /// + internal void ReadGroup(int fieldNumber, UnknownFieldSet set) + { + if (recursionDepth >= recursionLimit) + { + throw InvalidProtocolBufferException.RecursionLimitExceeded(); + } + ++recursionDepth; + set.MergeGroupFrom(this); + CheckLastTagWas(WireFormat.MakeTag(fieldNumber, WireFormat.WireType.EndGroup)); --recursionDepth; } diff --git a/csharp/src/Google.Protobuf/UnknownFieldSet.cs b/csharp/src/Google.Protobuf/UnknownFieldSet.cs index d136cf1e65..7a2b6a00d2 100644 --- a/csharp/src/Google.Protobuf/UnknownFieldSet.cs +++ b/csharp/src/Google.Protobuf/UnknownFieldSet.cs @@ -215,12 +215,8 @@ namespace Google.Protobuf } case WireFormat.WireType.StartGroup: { - uint endTag = WireFormat.MakeTag(number, WireFormat.WireType.EndGroup); UnknownFieldSet set = new UnknownFieldSet(); - while (input.ReadTag() != endTag) - { - set.MergeFieldFrom(input); - } + input.ReadGroup(number, set); GetOrAddField(number).AddGroup(set); return true; } @@ -233,6 +229,22 @@ namespace Google.Protobuf } } + internal void MergeGroupFrom(CodedInputStream input) + { + while (true) + { + uint tag = input.ReadTag(); + if (tag == 0) + { + break; + } + if (!MergeFieldFrom(input)) + { + break; + } + } + } + /// /// Create a new UnknownFieldSet if unknownFields is null. /// Parse a single field from and merge it From e9e4c687d6fdc79c3831c4b432166c67d27c7247 Mon Sep 17 00:00:00 2001 From: Rafi Kamal Date: Fri, 14 Feb 2020 10:19:40 -0800 Subject: [PATCH 23/27] Update CHANGES.txt to include the description of #7210 (#7212) --- CHANGES.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.txt b/CHANGES.txt index 3da8143a6f..67c5a3d6c7 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -2,6 +2,7 @@ C# * Fix latest ArgumentException for C# extensions (#7188) + * Enforce recursion depth checking for unknown fields (#7210) Ruby * Fix wrappers with a zero value (#7195) From d0bfd5221182da1a7cc280f3337b5e41a89539cf Mon Sep 17 00:00:00 2001 From: Rafi Kamal Date: Fri, 14 Feb 2020 12:13:20 -0800 Subject: [PATCH 24/27] Skip building distribution tar files for 32 bit Mac (#7214) --- protoc-artifacts/build-zip.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/protoc-artifacts/build-zip.sh b/protoc-artifacts/build-zip.sh index 7c4daf47ee..b5bf70a5bc 100755 --- a/protoc-artifacts/build-zip.sh +++ b/protoc-artifacts/build-zip.sh @@ -13,10 +13,9 @@ Example: This script will download pre-built protoc or protoc plugin binaries from maven repository and create .zip packages suitable to be included in the github release page. If the target is protoc, well-known type .proto files will also be -included. Each invocation will create 9 zip packages: +included. Each invocation will create 8 zip packages: dist/--win32.zip dist/--win64.zip - dist/--osx-x86_32.zip dist/--osx-x86_64.zip dist/--linux-x86_32.zip dist/--linux-x86_64.zip @@ -34,7 +33,6 @@ VERSION_NUMBER=$2 declare -a FILE_NAMES=( \ win32.zip windows-x86_32.exe \ win64.zip windows-x86_64.exe \ - osx-x86_32.zip osx-x86_32.exe \ osx-x86_64.zip osx-x86_64.exe \ linux-x86_32.zip linux-x86_32.exe \ linux-x86_64.zip linux-x86_64.exe \ From 1f2d6bf4b5f6e7cada0c3598a3c64b2966ebb28e Mon Sep 17 00:00:00 2001 From: Rafi Kamal Date: Fri, 14 Feb 2020 12:50:45 -0800 Subject: [PATCH 25/27] Remove 32-bit Mac binary from csharp build tools (#7215) --- csharp/build_tools.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/csharp/build_tools.sh b/csharp/build_tools.sh index 182c5c5c82..771affdc40 100755 --- a/csharp/build_tools.sh +++ b/csharp/build_tools.sh @@ -19,7 +19,6 @@ VERSION_NUMBER=$1 declare -a FILE_NAMES=( \ windows_x86 windows-x86_32.exe \ windows_x64 windows-x86_64.exe \ - macosx_x86 osx-x86_32.exe \ macosx_x64 osx-x86_64.exe \ linux_x86 linux-x86_32.exe \ linux_x64 linux-x86_64.exe \ From 147fd0b710083e3aaa96b65505784fff600a3c50 Mon Sep 17 00:00:00 2001 From: Rafi Kamal Date: Tue, 18 Feb 2020 10:23:20 -0800 Subject: [PATCH 26/27] Update Google.Protobuf.Tools.nuspec to remove 32-bit Mac protoc reference (#7227) --- csharp/Google.Protobuf.Tools.nuspec | 1 - 1 file changed, 1 deletion(-) diff --git a/csharp/Google.Protobuf.Tools.nuspec b/csharp/Google.Protobuf.Tools.nuspec index d8881869fa..91d28ed140 100644 --- a/csharp/Google.Protobuf.Tools.nuspec +++ b/csharp/Google.Protobuf.Tools.nuspec @@ -20,7 +20,6 @@ - From df2bce345d4bc8cdc3eba2a866e11e79e1fff4df Mon Sep 17 00:00:00 2001 From: Rafi Kamal Date: Tue, 18 Feb 2020 12:55:01 -0800 Subject: [PATCH 27/27] Remove 32-bit Mac protoc reference from collect_all_artifacts (#7228) --- kokoro/release/collect_all_artifacts.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/kokoro/release/collect_all_artifacts.sh b/kokoro/release/collect_all_artifacts.sh index 0023937d53..3b7d7d4066 100755 --- a/kokoro/release/collect_all_artifacts.sh +++ b/kokoro/release/collect_all_artifacts.sh @@ -43,9 +43,7 @@ mkdir -p protoc/linux_x64 cp ${INPUT_ARTIFACTS_DIR}/protoc-artifacts/target/linux/x86_32/protoc.exe protoc/linux_x86/protoc cp ${INPUT_ARTIFACTS_DIR}/protoc-artifacts/target/linux/x86_64/protoc.exe protoc/linux_x64/protoc -mkdir -p protoc/macosx_x86 mkdir -p protoc/macosx_x64 -cp ${INPUT_ARTIFACTS_DIR}/build32/src/protoc protoc/macosx_x86/protoc cp ${INPUT_ARTIFACTS_DIR}/build64/src/protoc protoc/macosx_x64/protoc # Install nuget (will also install mono)