Sync from Piper @461667700

PROTOBUF_SYNC_PIPER

CHANGES.txt

@@ -1,19 +1,18 @@
 2022-07-01 Unreleased version
-  * Add TransferAllMessages() util function
-  * Break ZeroCopyOutputByteSink into its own file.
-  * Explicitly instantiate InternalMetadata members at UnknownFieldSet.
-  * Support kdoc for Kotlin
-  * Delete unused function: IsProto3Field.
+  C++
+  * Reduced .pb.o object file size slightly by explicitly instantiating
+    InternalMetadata templates in the runtime.
   * Add C++20 keywords guarded by PROTOBUF_FUTURE_CPP20_KEYWORDS
-  * Escape Kotlin keywords in package names in proto generated code
-  * Fix StrAppend crashing on empty strings.
-
-2022-06-27 Unreleased version
-  * Handle reflection for message splitting.
-  * make metadata fields lazy.
-  * Extend visibility of plugin library to upb
-  * Modernize conformance_cpp.cc.
-  * Don't request 64-byte alignment unless the toolchain supports it.
+  * Fixed crash in ThreadLocalStorage for pre-C++17 compilers on 32-bit ARM.
+  * Clarified that JSON API non-OK statuses are not a stable API.
+
+  Kotlin
+  * Kotlin generated code comments now use kdoc format instead of javadoc.
+  * Escape keywords in package names in proto generated code
+
+  Java
+  * Performance improvement for repeated use of FieldMaskUtil#merge by caching
+    constructed FieldMaskTrees.
 
 2022-06-27 version 21.2 (C++/Java/Python/PHP/Objective-C/C#/Ruby)
   C++

Makefile.am

@@ -193,6 +193,7 @@ csharp_EXTRA_DIST=                                                           \
   csharp/src/Google.Protobuf/Compatibility/RequiresUnreferencedCodeAttribute.cs \
   csharp/src/Google.Protobuf/Compatibility/TypeExtensions.cs                 \
   csharp/src/Google.Protobuf/Compatibility/UnconditionalSuppressMessageAttribute.cs \
+  csharp/src/Google.Protobuf/Compiler/Plugin.cs                              \
   csharp/src/Google.Protobuf/Extension.cs                                    \
   csharp/src/Google.Protobuf/ExtensionRegistry.cs                            \
   csharp/src/Google.Protobuf/ExtensionSet.cs                                 \

README.md

@@ -82,3 +82,10 @@ The complete documentation for Protocol Buffers is available via the
 web at:
 
 https://developers.google.com/protocol-buffers/
+
+Developer Community
+-------------------
+
+To be alerted to upcoming changes in Protocol Buffers and connect with protobuf developers and users,
+[join the Google Group](https://groups.google.com/g/protobuf).
+

conformance/binary_json_conformance_suite.cc

@@ -2724,6 +2724,12 @@ void BinaryAndJsonConformanceSuite::RunJsonTestsForWrapperTypes() {
       "DurationNull", REQUIRED,
       R"({"optionalDuration": null})",
       "");
+  RunValidJsonTest("DurationNegativeSeconds", REQUIRED,
+                   R"({"optionalDuration": "-5s"})",
+                   "optional_duration: {seconds: -5 nanos: 0}");
+  RunValidJsonTest("DurationNegativeNanos", REQUIRED,
+                   R"({"optionalDuration": "-0.5s"})",
+                   "optional_duration: {seconds: 0 nanos: -500000000}");
 
   ExpectParseFailureForJson(
       "DurationMissingS", REQUIRED,

csharp/README.md

@@ -16,18 +16,19 @@ To generate C# files from your `.proto` files, invoke `protoc` with the
 Supported platforms
 ===================
 
-The runtime library is built as a portable class library, supporting:
+The runtime library is built as a class library, supporting targets of:
 
-- .NET 4.5
-- Windows 8
-- Windows Phone Silverlight 8
-- Windows Phone 8.1
-- .NET Core
-- .NET 5
+- .NET 4.5+ (`net45`)
+- .NET Standard 1.1 and 2.0 (`netstandard1.1` and `netstandard2.0`)
+- .NET 5+ (`net50`)
 
 You should be able to use Protocol Buffers in Visual Studio 2012 and
 all later versions. This includes all code generated by `protoc`,
-which only uses features from C# 3 and earlier.
+which only uses features from C# 3 and earlier. When compiling generated
+code with old compilers (before C# 7.2) you need to define the
+`GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE` symbol in your project
+so that the generated classes don't implement `IBufferMessage`, which uses
+`ref struct` types.
 
 Building
 ========
@@ -42,12 +43,6 @@ in its implementation and runs tests under .NET 6. These features
 have no impact when using the compiled code - they're only relevant
 when building the `Google.Protobuf` assembly.
 
-In order to run and debug the AddressBook example in the IDE, you must
-install the optional component, ".Net Core 1.0 - 1.1 development tools
-for Web" (as it's labelled in current versions of the VS2017
-installer), above and beyond the main .NET Core cross-platform
-development feature.
-
 Testing
 =======
 
@@ -57,20 +52,11 @@ run using the Visual Studio Test Explorer or `dotnet test`.
 .NET 3.5
 ========
 
-We don't officially support .NET 3.5. However, there has been some effort
-to make enabling .NET 3.5 support relatively painless in case you require it.
-There's no guarantee that this will continue in the future, so rely on .NET
-3.5 support at your peril.
-
-To enable .NET 3.5 support, you must edit the `TargetFrameworks` elements of
-[src/Google.Protobuf/Google.Protobuf.csproj](src/Google.Protobuf/Google.Protobuf.csproj)
-(and [src/Google.Protobuf.Test/Google.Protobuf.Test.csproj](src/Google.Protobuf.Test/Google.Protobuf.Test.csproj)
-if you want to run the unit tests):
-
-Open the .csproj file in a text editor and simply add `net35` to the list of
-target frameworks, noting that the `TargetFrameworks` element appears twice in
-the file (once in the first `PropertyGroup` element, and again in the second
-`PropertyGroup` element, i.e., the one with the conditional).
+We don't support .NET 3.5. It *used* to be feasible to build this library
+targeting .NET 3.5, but a number of changes requiring newer runtime/framework
+features have been added over time. While it would no doubt be *possible* to
+rework the current implementation to allow most of the functionality to be built
+in .NET 3.5, this would create an undue maintenance burden.
 
 History of C# protobufs
 =======================
@@ -82,9 +68,9 @@ and maintained by Google. All the development will be done in open, under this r
 
 The previous project differs from this project in a number of ways:
 
-- The old code only supported proto2; the new code only supports
+- The old code only supported proto2; the new code initially only supported
 proto3 (so no unknown fields, no required/optional distinction, no
-extensions)
+extensions); since then proto2 support has been added
 - The old code was based on immutable message types and builders for
 them
 - The old code did not support maps or `oneof`

csharp/generate_protos.sh

@@ -40,7 +40,8 @@ $PROTOC -Isrc --csharp_out=csharp/src/Google.Protobuf \
     src/google/protobuf/struct.proto \
     src/google/protobuf/timestamp.proto \
     src/google/protobuf/type.proto \
-    src/google/protobuf/wrappers.proto
+    src/google/protobuf/wrappers.proto \
+    src/google/protobuf/compiler/plugin.proto
 
 # Test protos
 # Note that this deliberately does *not* include old_extensions1.proto

java/core/src/main/java/com/google/protobuf/Descriptors.java

@@ -1788,6 +1788,27 @@ public final class Descriptors {
       return Collections.unmodifiableList(Arrays.asList(values));
     }
 
+    /** Determines if the given field number is reserved. */
+    public boolean isReservedNumber(final int number) {
+      for (final EnumDescriptorProto.EnumReservedRange range : proto.getReservedRangeList()) {
+        if (range.getStart() <= number && number <= range.getEnd()) {
+          return true;
+        }
+      }
+      return false;
+    }
+
+    /** Determines if the given field name is reserved. */
+    public boolean isReservedName(final String name) {
+      checkNotNull(name);
+      for (final String reservedName : proto.getReservedNameList()) {
+        if (reservedName.equals(name)) {
+          return true;
+        }
+      }
+      return false;
+    }
+
     /**
      * Find an enum value by name.
      *

java/core/src/test/java/com/google/protobuf/DescriptorsTest.java

@@ -61,6 +61,7 @@ import protobuf_unittest.UnittestProto.TestExtremeDefaultValues;
 import protobuf_unittest.UnittestProto.TestJsonName;
 import protobuf_unittest.UnittestProto.TestMultipleExtensionRanges;
 import protobuf_unittest.UnittestProto.TestRequired;
+import protobuf_unittest.UnittestProto.TestReservedEnumFields;
 import protobuf_unittest.UnittestProto.TestReservedFields;
 import protobuf_unittest.UnittestProto.TestService;
 import java.util.Collections;
@@ -796,6 +797,20 @@ public class DescriptorsTest {
     assertThat(d.isReservedName("baz")).isTrue();
   }
 
+  @Test
+  public void testReservedEnumFields() {
+    EnumDescriptor d = TestReservedEnumFields.getDescriptor();
+    assertThat(d.isReservedNumber(2)).isTrue();
+    assertThat(d.isReservedNumber(8)).isFalse();
+    assertThat(d.isReservedNumber(9)).isTrue();
+    assertThat(d.isReservedNumber(10)).isTrue();
+    assertThat(d.isReservedNumber(11)).isTrue();
+    assertThat(d.isReservedNumber(12)).isFalse();
+    assertThat(d.isReservedName("foo")).isFalse();
+    assertThat(d.isReservedName("bar")).isTrue();
+    assertThat(d.isReservedName("baz")).isTrue();
+  }
+
   @Test
   public void testToString() {
     assertThat(

src/google/protobuf/compiler/plugin.proto

@@ -30,9 +30,6 @@
 
 // Author: kenton@google.com (Kenton Varda)
 //
-// WARNING:  The plugin interface is currently EXPERIMENTAL and is subject to
-//   change.
-//
 // protoc (aka the Protocol Compiler) can be extended via plugins.  A plugin is
 // just a program that reads a CodeGeneratorRequest from stdin and writes a
 // CodeGeneratorResponse to stdout.

src/google/protobuf/port_def.inc

@@ -154,6 +154,13 @@
 #  define PROTOBUF_GNUC_MIN(x, y) 0
 #endif
 
+#if defined(__clang__) && defined(__clang_major__) && defined(__clang_minor__)
+#define PROTOBUF_CLANG_MIN(x, y) \
+  (__clang_major__ > (x) || __clang_major__ == (x) && __clang_minor__ >= (y))
+#else
+#define PROTOBUF_CLANG_MIN(x, y) 0
+#endif
+
 // Portable check for MSVC minimum version:
 // https://docs.microsoft.com/en-us/cpp/preprocessor/predefined-macros
 #if defined(_MSC_VER)
@@ -666,30 +673,37 @@
 // This experiment is purely for the purpose of gathering data. All code guarded
 // by this flag is supposed to be removed after this experiment.
 #define PROTOBUF_MESSAGE_OWNED_ARENA_EXPERIMENT
+
 #ifdef PROTOBUF_CONSTINIT
 #error PROTOBUF_CONSTINIT was previously defined
 #endif
-#if defined(__cpp_constinit) && !defined(_MSC_VER)
-#define PROTOBUF_CONSTINIT constinit
-#define PROTOBUF_CONSTEXPR constexpr
+
+#if defined(_MSC_VER)
+// MSVC 17 currently seems to raise an error about constant-initialized pointers.
+# if PROTOBUF_MSC_VER_MIN(1930)
+#  define PROTOBUF_CONSTINIT
+#  define PROTOBUF_CONSTEXPR constexpr
+# endif
+#else
+# if defined(__cpp_constinit)
+#  define PROTOBUF_CONSTINIT constinit
+#  define PROTOBUF_CONSTEXPR constexpr
 // Some older Clang versions incorrectly raise an error about
 // constant-initializing weak default instance pointers. Versions 12.0 and
 // higher seem to work, except that XCode 12.5.1 shows the error even though it
 // uses Clang 12.0.5.
-// Clang-cl on Windows raises error also.
-#elif !defined(_MSC_VER) && __has_cpp_attribute(clang::require_constant_initialization) && \
-    ((defined(__APPLE__) && __clang_major__ >= 13) ||                \
-     (!defined(__APPLE__) && __clang_major__ >= 12))
-#define PROTOBUF_CONSTINIT [[clang::require_constant_initialization]]
-#define PROTOBUF_CONSTEXPR constexpr
-#elif PROTOBUF_GNUC_MIN(12, 2)
-#define PROTOBUF_CONSTINIT __constinit
-#define PROTOBUF_CONSTEXPR constexpr
-// MSVC 17 currently seems to raise an error about constant-initialized pointers.
-#elif defined(_MSC_VER) && _MSC_VER >= 1930
-#define PROTOBUF_CONSTINIT
-#define PROTOBUF_CONSTEXPR constexpr
-#else
+# elif __has_cpp_attribute(clang::require_constant_initialization) && \
+    ((defined(__APPLE__) && PROTOBUF_CLANG_MIN(13, 0)) ||             \
+     (!defined(__APPLE__) && PROTOBUF_CLANG_MIN(12, 0)))
+#  define PROTOBUF_CONSTINIT [[clang::require_constant_initialization]]
+#  define PROTOBUF_CONSTEXPR constexpr
+# elif PROTOBUF_GNUC_MIN(12, 2)
+#  define PROTOBUF_CONSTINIT __constinit
+#  define PROTOBUF_CONSTEXPR constexpr
+# endif
+#endif
+
+#ifndef PROTOBUF_CONSTINIT
 #define PROTOBUF_CONSTINIT
 #define PROTOBUF_CONSTEXPR
 #endif

src/google/protobuf/port_undef.inc

@@ -40,6 +40,7 @@
 #undef PROTOBUF_BUILTIN_BSWAP64
 #undef PROTOBUF_BUILTIN_ATOMIC
 #undef PROTOBUF_GNUC_MIN
+#undef PROTOBUF_CLANG_MIN
 #undef PROTOBUF_MSC_VER_MIN
 #undef PROTOBUF_CPLUSPLUS_MIN
 #undef PROTOBUF_NAMESPACE

src/google/protobuf/unittest.proto

@@ -221,6 +221,12 @@ message TestReservedFields {
   reserved "bar", "baz";
 }
 
+enum TestReservedEnumFields {
+  UNKNOWN = 0;
+  reserved 2, 15, 9 to 11;
+  reserved "bar", "baz";
+}
+
 message TestAllExtensions {
   extensions 1 to max;
 }

src/google/protobuf/util/json_util.cc

@@ -48,10 +48,12 @@
 #include <google/protobuf/util/zero_copy_sink.h>
 #include <google/protobuf/stubs/status_macros.h>
 
+
 // clang-format off
 #include <google/protobuf/port_def.inc>
 // clang-format on
 
+
 namespace google {
 namespace protobuf {
 namespace util {
@@ -62,6 +64,7 @@ util::Status BinaryToJsonStream(TypeResolver* resolver,
                                 io::ZeroCopyInputStream* binary_input,
                                 io::ZeroCopyOutputStream* json_output,
                                 const JsonPrintOptions& options) {
+
   io::CodedInputStream in_stream(binary_input);
   google::protobuf::Type type;
   RETURN_IF_ERROR(resolver->ResolveMessageType(type_url, &type));
@@ -152,6 +155,7 @@ util::Status JsonToBinaryStream(TypeResolver* resolver,
                                 io::ZeroCopyInputStream* json_input,
                                 io::ZeroCopyOutputStream* binary_output,
                                 const JsonParseOptions& options) {
+
   google::protobuf::Type type;
   RETURN_IF_ERROR(resolver->ResolveMessageType(type_url, &type));
   ZeroCopyStreamByteSink sink(binary_output);
@@ -219,6 +223,7 @@ TypeResolver* GetGeneratedTypeResolver() {
 
 util::Status MessageToJsonString(const Message& message, std::string* output,
                                  const JsonOptions& options) {
+
   const DescriptorPool* pool = message.GetDescriptor()->file()->pool();
   TypeResolver* resolver =
       pool == DescriptorPool::generated_pool()
@@ -235,6 +240,7 @@ util::Status MessageToJsonString(const Message& message, std::string* output,
 
 util::Status JsonStringToMessage(StringPiece input, Message* message,
                                  const JsonParseOptions& options) {
+
   const DescriptorPool* pool = message->GetDescriptor()->file()->pool();
   TypeResolver* resolver =
       pool == DescriptorPool::generated_pool()

src/google/protobuf/util/json_util_test.cc

@@ -470,8 +470,10 @@ TEST_P(JsonTest, ParseMap) {
 
 TEST_P(JsonTest, RepeatedMapKey) {
   EXPECT_THAT(ToProto<TestMap>(R"json({
-    "twiceKey": 0,
-    "twiceKey": 1
+    "string_map": {
+      "twiceKey": 0,
+      "twiceKey": 1
+    }
   })json"), StatusIs(util::StatusCode::kInvalidArgument));
 }
 
@@ -700,6 +702,37 @@ TEST_P(JsonTest, TestParsingNestedAnys) {
           R"("int32Value":5,"stringValue":"expected_value","messageValue":{"value":1}}}})"));
 }
 
+TEST_P(JsonTest, TestParsingBrokenAny) {
+  auto m = ToProto<TestAny>(R"json(
+    {
+      "value": {}
+    }
+  )json");
+  ASSERT_OK(m);
+  EXPECT_EQ(m->value().type_url(), "");
+  EXPECT_EQ(m->value().value(), "");
+
+  EXPECT_THAT(ToProto<TestAny>(R"json(
+    {
+      "value": {
+        "type_url": "garbage",
+        "value": "bW9yZSBnYXJiYWdl"
+      }
+    }
+  )json"),
+              StatusIs(util::StatusCode::kInvalidArgument));
+}
+
+TEST_P(JsonTest, TestFlatList) {
+  auto m = ToProto<TestMessage>(R"json(
+    {
+      "repeatedInt32Value": [[[5]], [6]]
+    }
+  )json");
+  ASSERT_OK(m);
+  EXPECT_THAT(m->repeated_int32_value(), ElementsAre(5, 6));
+}
+
 TEST_P(JsonTest, ParseWrappers) {
   auto m = ToProto<TestWrapper>(R"json(
     {
@@ -757,6 +790,14 @@ TEST_P(JsonTest, TestParsingUnknownAnyFields) {
   EXPECT_EQ(t.string_value(), "expected_value");
 }
 
+TEST_P(JsonTest, TestHugeBareString) {
+  auto m = ToProto<TestMessage>(R"json({
+    "int64Value": 6009652459062546621
+  })json");
+  ASSERT_OK(m);
+  EXPECT_EQ(m->int64_value(), 6009652459062546621);
+}
+
 TEST_P(JsonTest, TestParsingUnknownEnumsProto2) {
   StringPiece input = R"json({"ayuLmao": "UNKNOWN_VALUE"})json";
 
@@ -895,6 +936,39 @@ TEST_P(JsonTest, TestDuration) {
 
   EXPECT_EQ(m2->value().seconds(), 4);
   EXPECT_EQ(m2->value().nanos(), 5);
+
+  // Negative duration with zero seconds.
+  auto m3 = ToProto<proto3::TestDuration>(R"json(
+    {
+      "value": {"nanos": -5},
+    }
+  )json");
+  ASSERT_OK(m3);
+  EXPECT_EQ(m3->value().seconds(), 0);
+  EXPECT_EQ(m3->value().nanos(), -5);
+  EXPECT_THAT(ToJson(m3->value()), IsOkAndHolds("\"-0.000000005s\""));
+
+  // Negative duration with zero nanos.
+  auto m4 = ToProto<proto3::TestDuration>(R"json(
+    {
+      "value": {"seconds": -5},
+    }
+  )json");
+  ASSERT_OK(m4);
+  EXPECT_EQ(m4->value().seconds(), -5);
+  EXPECT_EQ(m4->value().nanos(), 0);
+  EXPECT_THAT(ToJson(m4->value()), IsOkAndHolds("\"-5s\""));
+
+  // Parse "0.5s" as a JSON string.
+  auto m5 = ToProto<proto3::TestDuration>(R"json(
+    {
+      "value": "0.5s",
+    }
+  )json");
+  ASSERT_OK(m5);
+  EXPECT_EQ(m5->value().seconds(), 0);
+  EXPECT_EQ(m5->value().nanos(), 500000000);
+  EXPECT_THAT(ToJson(m5->value()), IsOkAndHolds("\"0.500s\""));
 }
 
 // These tests are not exhaustive; tests in //third_party/protobuf/conformance