From e5fe9b8cbdfe7d25852ff2e34e3250e2c0c98860 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Mon, 17 Aug 2020 12:58:45 -0700 Subject: [PATCH 01/14] Updated CHANGES.txt Now that 3.13.0 has been released, this commit updates CHANGES.txt to mention 3.13.0 instead of 3.13.0-rc1 and to include the recent PHP changes. I also added a short explanation of the 3.12.4 release. Fixes #7820. --- CHANGES.txt | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 8c1bc2fc08..511d84e882 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -33,7 +33,13 @@ Unreleased Changes * Update go_package options to reference google.golang.org/protobuf module. -2020-07-14 version 3.13.0-rc1 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) +2020-07-14 version 3.13.0 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) + + PHP: + * The C extension is completely rewritten. The new C extension has significantly + better parsing performance and fixes a handful of conformance issues. It will + also make it easier to add support for more features like proto2 and proto3 presence. + * The new C extension does not support PHP 5.x. PHP 5.x users can still use pure-PHP. C++: * Removed deprecated unsafe arena string accessors @@ -86,6 +92,11 @@ Unreleased Changes performance (the legacy generated code will still work, but might incur a slight performance penalty). +2020-07-28 version 3.12.4 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) + +This release contains no significant changes, but exists because 3.12.3 was +mistakenly tagged at the wrong commit. + 2020-06-01 version 3.12.3 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) Objective-C From 7e787e091e1e600b5dc3523269b988b952a0e9db Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Tue, 18 Aug 2020 11:28:31 -0700 Subject: [PATCH 02/14] Made the Codespell check skip protobuf generated code --- .github/workflows/codespell.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index d853702a11..4ef1a1bd27 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -12,5 +12,5 @@ jobs: - uses: codespell-project/actions-codespell@master with: check_filenames: true - skip: ./.git,./conformance/third_party,*.snk,*.pb,./src/google/protobuf/testdata,./objectivec/Tests,./python/compatibility_tests/v2.5.0/tests/google/protobuf/internal + skip: ./.git,./conformance/third_party,*.snk,*.pb,*.pb.cc,*.pb.h,./src/google/protobuf/testdata,./objectivec/Tests,./python/compatibility_tests/v2.5.0/tests/google/protobuf/internal ignore_words_list: "alow,alse,ba,cleare,copyable,cloneable,dedup,dur,errorprone,fo,fundementals,hel,importd,inout,leapyear,nd,ois,ons,parseable,process',te,testof,ue,unparseable,wasn,wee,gae,keyserver,objext,od" From b970be43ffb4f6e9c7248c5972d984551ed59fff Mon Sep 17 00:00:00 2001 From: Saurabh Singhal Date: Wed, 12 Aug 2020 21:38:44 +0530 Subject: [PATCH 03/14] Display beta version in "protoc --version" Display the beta version when "protoc --version" is executed. Use PROTOBUF_VERSION_SUFFIX for this purpose. --- src/google/protobuf/compiler/command_line_interface.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/google/protobuf/compiler/command_line_interface.cc b/src/google/protobuf/compiler/command_line_interface.cc index 3d5c9dee56..905981c4cb 100644 --- a/src/google/protobuf/compiler/command_line_interface.cc +++ b/src/google/protobuf/compiler/command_line_interface.cc @@ -1842,7 +1842,7 @@ CommandLineInterface::InterpretArgument(const std::string& name, std::cout << version_info_ << std::endl; } std::cout << "libprotoc " << internal::VersionString(PROTOBUF_VERSION) - << std::endl; + << PROTOBUF_VERSION_SUFFIX << std::endl; return PARSE_ARGUMENT_DONE_AND_EXIT; // Exit without running compiler. } else if (name == "--disallow_services") { From 7db720e07524f96be558f194ae1ffee4a9d49972 Mon Sep 17 00:00:00 2001 From: Saurabh Singhal Date: Fri, 14 Aug 2020 22:36:12 +0530 Subject: [PATCH 04/14] Modify update_version.py to update the version suffix as well. --- update_version.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/update_version.py b/update_version.py index 1f3332d143..10410e075d 100755 --- a/update_version.py +++ b/update_version.py @@ -110,6 +110,15 @@ def UpdateCpp(): r'^#define PROTOBUF_VERSION .*$', '#define PROTOBUF_VERSION %s' % cpp_version, line) + if RC_VERSION != -1: + line = re.sub( + r'^#define GOOGLE_PROTOBUF_VERSION_SUFFIX .*$', + '#define GOOGLE_PROTOBUF_VERSION_SUFFIX %s' % RC_VERSION, + line) + line = re.sub( + r'^#define PROTOBUF_VERSION_SUFFIX .*$', + '#define PROTOBUF_VERSION_SUFFIX %s' % RC_VERSION, + line) if NEW_VERSION_INFO[2] == 0: line = re.sub( r'^#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC .*$', @@ -134,6 +143,11 @@ def UpdateCpp(): r'^#define PROTOBUF_VERSION .*$', '#define PROTOBUF_VERSION %s' % cpp_version, line) + if RC_VERSION != -1: + line = re.sub( + r'^#define PROTOBUF_VERSION_SUFFIX .*$', + '#define PROTOBUF_VERSION_SUFFIX %s' % RC_VERSION, + line) if NEW_VERSION_INFO[2] == 0: line = re.sub( r'^#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC .*$', From 543817295e05bcf226628ec26706e575741a0722 Mon Sep 17 00:00:00 2001 From: Mathijs Vogelzang Date: Wed, 5 Aug 2020 13:27:51 +0200 Subject: [PATCH 05/14] Add R8 rule to documentation Using default production Android build and protobuf-lite leads to RuntimeExceptions. The workarounds are documented in issues and I took those discussions and summarized them in the lite.md file. --- java/lite.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/java/lite.md b/java/lite.md index 5cbfbb1d5f..f248ff6c3e 100644 --- a/java/lite.md +++ b/java/lite.md @@ -34,6 +34,22 @@ protobuf Java runtime. If you are using Maven, use the following: ``` +## R8 rule to make production app builds work + +The Lite runtime internally uses reflection to avoid generating hashCode/equals/(de)serialization methods. +R8 by default obfuscates the field names, which makes the reflection fail causing exceptions of the form +`java.lang.RuntimeException: Field {NAME}_ for {CLASS} not found. Known fields are [ {FIELDS} ]` in MessageSchema.java. + +There are open issues for this on the [protobuf Github project](https://github.com/protocolbuffers/protobuf/issues/6463) and [R8](https://issuetracker.google.com/issues/144631039). + +Until the issues is resolved you need to add the following line to your `proguard-rules.pro` file inside your project: + +``` +-keep class * extends com.google.protobuf.GeneratedMessageLite { *; } +``` + +## Older versions + For the older version of Java Lite (v3.0.0), please refer to: https://github.com/protocolbuffers/protobuf/blob/javalite/java/lite.md From 88f3ef7d1dd3557d22782094b9ed4bf8a5010dbe Mon Sep 17 00:00:00 2001 From: Adam Yi Date: Wed, 29 Jul 2020 16:49:52 +1000 Subject: [PATCH 06/14] [bazel] Add default shell env to gen_well_known_protos_java We may need the shell environment (potentially augmented with `--action_env`) to invoke protoc on Windows. If protoc was built with mingw, it probably needs .dll files in non-default locations that must be in PATH. Previously with `--compiler=mingw-gcc`, `bazel build //:gen_well_known_protos_java` would fail on Windows. This CL fixes the issue. Also we have `default_shell_env` set to True for `ProtoCompile`, this makes the behavior consistent. See #2933, and https://github.com/bazelbuild/rules_go/blob/585a27ad0ab5bdd185aa3bd5b0877a778d4777ad/proto/compiler.bzl#L130 --- protobuf.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/protobuf.bzl b/protobuf.bzl index 050eafc548..12d3edb947 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -352,6 +352,7 @@ def _internal_gen_well_known_protos_java_impl(ctx): inputs = descriptors, outputs = [srcjar], arguments = [args], + use_default_shell_env = True, ) return [ From a446e6472b87e2c06056773ba28cbf6e8bd117c1 Mon Sep 17 00:00:00 2001 From: Afshin Pir Date: Wed, 29 Jul 2020 21:19:56 +0430 Subject: [PATCH 07/14] Previously, I had incorrectly used some newer features of CMake(version 3.14). But Now I have updated this code to work with older versions of CMake too. --- cmake/protobuf-config.cmake.in | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmake/protobuf-config.cmake.in b/cmake/protobuf-config.cmake.in index 11b85d3c07..fac5efe440 100644 --- a/cmake/protobuf-config.cmake.in +++ b/cmake/protobuf-config.cmake.in @@ -99,13 +99,17 @@ function(protobuf_generate) foreach(_proto ${protobuf_generate_PROTOS}) get_filename_component(_abs_file ${_proto} ABSOLUTE) get_filename_component(_abs_dir ${_abs_file} DIRECTORY) - get_filename_component(_basename ${_proto} NAME_WLE) + + get_filename_component(_file_full_name ${_proto} NAME) + string(FIND "${_file_full_name}" "." _file_last_ext_pos REVERSE) + string(SUBSTRING "${_file_full_name}" 0 ${_file_last_ext_pos} _basename) set(_suitable_include_found FALSE) foreach(DIR ${_protobuf_include_path}) if(NOT DIR STREQUAL "-I") file(RELATIVE_PATH _rel_dir ${DIR} ${_abs_dir}) - if(NOT "${_rel_dir}" MATCHES "^\.\.[/\\].*") + string(FIND "${_rel_dir}" "../" _is_in_parent_folder) + if (NOT ${_is_in_parent_folder} EQUAL 0) set(_suitable_include_found TRUE) break() endif() From 223e89157a8b8b4f9e57ee4a3cf2165c61372f5d Mon Sep 17 00:00:00 2001 From: Misha Seltzer Date: Thu, 14 May 2020 21:09:09 -0400 Subject: [PATCH 08/14] Removed test files from pypi distribution generation code. Fixes #7493 By removing all test files from the distribution, we've reduced the .whl file size from 1259808B (1.2MB) to 996042B (973K), which is 21% reduction, And reduced the unpacked size from 5317178B (5.1MB) to 3251811B (3.2MB), which is 39% reduction. Size was measured for the protobuf-3.11.3-cp35-cp35m-manylinux1_x86_64.whl release. --- python/setup.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/python/setup.py b/python/setup.py index cd66d6c773..d29ff22820 100755 --- a/python/setup.py +++ b/python/setup.py @@ -2,6 +2,7 @@ # # See README for usage instructions. from distutils import util +import fnmatch import glob import os import pkg_resources @@ -144,6 +145,18 @@ class build_py(_build_py): # _build_py is an old-style class, so super() doesn't work. _build_py.run(self) + def find_package_modules(self, package, package_dir): + exclude = ( + "*test*", + "google/protobuf/internal/*_pb2.py", + "google/protobuf/internal/_parameterized.py", + "google/protobuf/pyext/python_pb2.py", + ) + modules = _build_py.find_package_modules(self, package, package_dir) + return [(pkg, mod, fil) for (pkg, mod, fil) in modules + if not any(fnmatch.fnmatchcase(fil, pat=pat) for pat in exclude)] + + class test_conformance(_build_py): target = 'test_python' def run(self): From 090d28e8f75ff3c6f8d2996081e84b672fcfe808 Mon Sep 17 00:00:00 2001 From: CauhxMilloy Date: Thu, 20 Aug 2020 14:42:53 -0700 Subject: [PATCH 09/14] Null Check in UnknownField::AddAll() (#7328) UnknownField::AddAll() is called (multiple times) from UnknownField::MergeFrom(). The `extras` parameter is one of: `varintList`, `fixed32List`, `fixed64List`, `lengthDelimitedList`, or `groupList`. All of these members can be null, and are appropriately checked in other usage. If attempting to parse a proto with unknown extensions, and exception is thrown (NRE). This adds the appropriate null check inside UnknownField::AddAll(). --- csharp/src/Google.Protobuf/UnknownField.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/csharp/src/Google.Protobuf/UnknownField.cs b/csharp/src/Google.Protobuf/UnknownField.cs index 63d89604a2..4793a6419d 100644 --- a/csharp/src/Google.Protobuf/UnknownField.cs +++ b/csharp/src/Google.Protobuf/UnknownField.cs @@ -209,13 +209,13 @@ namespace Google.Protobuf /// /// Returns a new list containing all of the given specified values from /// both the and lists. - /// If is null and is empty, + /// If is null and is null or empty, /// null is returned. Otherwise, either a new list is created (if /// is null) or the elements of are added to . /// private static List AddAll(List current, IList extras) { - if (extras.Count == 0) + if (extras == null || extras.Count == 0) { return current; } From 4b93993ac714342e57aa3c47dfc07c9fb0e4c181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C5=91rinc=20Pap?= Date: Thu, 20 Aug 2020 18:45:45 +0200 Subject: [PATCH 10/14] Update Gradle dependency reference format See https://docs.gradle.org/current/userguide/java_library_plugin.html > The compile, testCompile, runtime and testRuntime configurations inherited from the Java plugin are still available but are deprecated. You should avoid using them, as they are only kept for backwards compatibility. --- java/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/README.md b/java/README.md index dea9e763f2..c8050893e0 100644 --- a/java/README.md +++ b/java/README.md @@ -45,7 +45,7 @@ protobuf-java-util package: If you are using Gradle, add the following to your `build.gradle` file's dependencies: ``` - compile 'com.google.protobuf:protobuf-java:3.11.0' + implementation 'com.google.protobuf:protobuf-java:3.11.0' ``` Again, be sure to check that the version number matches (or is newer than) the version number of protoc that you are using. From 6da79e94b0af7a82cffe8f65bc53911fbc44b5b6 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Wed, 19 Aug 2020 11:32:03 -0700 Subject: [PATCH 11/14] Fixed VERSION_SUFFIX logic in update_version.py This commit makes a couple of fixes: - Make sure we always update the suffix, since even for a non-RC release we need to clear the suffix. - Make sure the suffix is properly quoted and begins with -rc --- update_version.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/update_version.py b/update_version.py index 10410e075d..bfbef45ae1 100755 --- a/update_version.py +++ b/update_version.py @@ -101,6 +101,9 @@ def UpdateConfigure(): def UpdateCpp(): cpp_version = '%d%03d%03d' % ( NEW_VERSION_INFO[0], NEW_VERSION_INFO[1], NEW_VERSION_INFO[2]) + version_suffix = '' + if RC_VERSION != -1: + version_suffix = '-rc%s' % RC_VERSION def RewriteCommon(line): line = re.sub( r'^#define GOOGLE_PROTOBUF_VERSION .*$', @@ -110,15 +113,14 @@ def UpdateCpp(): r'^#define PROTOBUF_VERSION .*$', '#define PROTOBUF_VERSION %s' % cpp_version, line) - if RC_VERSION != -1: - line = re.sub( - r'^#define GOOGLE_PROTOBUF_VERSION_SUFFIX .*$', - '#define GOOGLE_PROTOBUF_VERSION_SUFFIX %s' % RC_VERSION, - line) - line = re.sub( - r'^#define PROTOBUF_VERSION_SUFFIX .*$', - '#define PROTOBUF_VERSION_SUFFIX %s' % RC_VERSION, - line) + line = re.sub( + r'^#define GOOGLE_PROTOBUF_VERSION_SUFFIX .*$', + '#define GOOGLE_PROTOBUF_VERSION_SUFFIX "%s"' % version_suffix, + line) + line = re.sub( + r'^#define PROTOBUF_VERSION_SUFFIX .*$', + '#define PROTOBUF_VERSION_SUFFIX "%s"' % version_suffix, + line) if NEW_VERSION_INFO[2] == 0: line = re.sub( r'^#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC .*$', @@ -137,17 +139,16 @@ def UpdateCpp(): 'static const int kMinHeaderVersionForProtoc = %s;' % cpp_version, line) return line - + def RewritePortDef(line): line = re.sub( r'^#define PROTOBUF_VERSION .*$', '#define PROTOBUF_VERSION %s' % cpp_version, line) - if RC_VERSION != -1: - line = re.sub( - r'^#define PROTOBUF_VERSION_SUFFIX .*$', - '#define PROTOBUF_VERSION_SUFFIX %s' % RC_VERSION, - line) + line = re.sub( + r'^#define PROTOBUF_VERSION_SUFFIX .*$', + '#define PROTOBUF_VERSION_SUFFIX "%s"' % version_suffix, + line) if NEW_VERSION_INFO[2] == 0: line = re.sub( r'^#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC .*$', From 1b18833f4f2a2f681f4e4a25cdf3b0a43115ec26 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Fri, 21 Aug 2020 11:12:57 -0700 Subject: [PATCH 12/14] Tweaked update_version.py to leave PHP notes blank Until now update_version.py has copied the notes from the previous PHP release, which is not very useful. --- update_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update_version.py b/update_version.py index bfbef45ae1..470e4251e6 100755 --- a/update_version.py +++ b/update_version.py @@ -317,7 +317,7 @@ def UpdatePhp(): FindAndClone(root, 'date'), FindAndClone(root, 'time'), FindAndClone(root, 'license'), - FindAndClone(root, 'notes') + CreateNode('notes', 3, []), ]) changelog.appendChild(release) changelog.appendChild(document.createTextNode('\n ')) From e1ee430d1619fcee4828ef4a14d7c03cebe95d23 Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Tue, 25 Aug 2020 14:28:24 -0700 Subject: [PATCH 13/14] Re-add fix that somehow was lost pushing back to Piper. --- benchmarks/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 436c148a34..76788175c6 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -135,7 +135,7 @@ $ make go ### PHP -We have two version of php protobuf implemention: pure php, php with c extension. To run these version benchmark, you need to: +We have two version of php protobuf implementation: pure php, php with c extension. To run these version benchmark, you need to: #### Pure PHP ``` $ make php From 5c26cae18c47ada3a4990c4bce33964c71ebed02 Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Tue, 25 Aug 2020 14:42:47 -0700 Subject: [PATCH 14/14] Re-add change that was somehow lost. --- python/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/setup.py b/python/setup.py index 902f1f4b76..d29ff22820 100755 --- a/python/setup.py +++ b/python/setup.py @@ -292,6 +292,7 @@ if __name__ == '__main__': 'build_py': build_py, 'test_conformance': test_conformance, }, + setup_requires = ['wheel'], install_requires=install_requires, ext_modules=ext_module_list, )