Sync from Piper @463670061

PROTOBUF_SYNC_PIPER

.gitignore

@@ -19,6 +19,23 @@ m4/ltversion.m4
 m4/lt~obsolete.m4
 autom4te.cache
 
+# CMake-generated files
+.ninja_deps
+.ninja_logs
+cmake/protobuf/*.cmake
+cmake_install.cmake
+CMakeCache.txt
+CTestTestfile.cmake
+CMakeFiles/*
+Testing/Temporary/*
+
+/core
+/protoc
+/test_plugin
+/tests
+/lite-test
+/protoc-*.*
+
 # downloaded files
 /gmock
 
@@ -41,6 +58,7 @@ stamp-h1
 *.la
 src/.libs
 *.so
+*.a
 
 .dirstamp
 

CHANGES.txt

@@ -1,4 +1,5 @@
 2022-07-01 Unreleased version
+
   C++
   * Reduced .pb.o object file size slightly by explicitly instantiating
     InternalMetadata templates in the runtime.
@@ -14,6 +15,12 @@
   * Performance improvement for repeated use of FieldMaskUtil#merge by caching
     constructed FieldMaskTrees.
 
+2022-07-25 version 21.4 (C++/Java/Python/PHP/Objective-C/C#/Ruby)
+
+  C++
+  * Reduce the required alignment of ArenaString from 8 to 4 (#10298)
+
+
 2022-07-19 version 21.3 (C++/Java/Python/PHP/Objective-C/C#/Ruby)
   C++
   * Add header search paths to Protobuf-C++.podspec (#10024)

Protobuf.podspec

@@ -5,7 +5,7 @@
 # dependent projects use the :git notation to refer to the library.
 Pod::Spec.new do |s|
   s.name     = 'Protobuf'
-  s.version  = '3.21.3'
+  s.version  = '3.21.4'
   s.summary  = 'Protocol Buffers v.3 runtime library for Objective-C.'
   s.homepage = 'https://github.com/protocolbuffers/protobuf'
   s.license  = 'BSD-3-Clause'

csharp/Google.Protobuf.Tools.nuspec

@@ -5,7 +5,7 @@
     <title>Google Protocol Buffers tools</title>
     <summary>Tools for Protocol Buffers - Google's data interchange format.</summary>
     <description>See project site for more info.</description>
-    <version>3.21.3</version>
+    <version>3.21.4</version>
     <authors>Google Inc.</authors>
     <owners>protobuf-packages</owners>
     <licenseUrl>https://github.com/protocolbuffers/protobuf/blob/main/LICENSE</licenseUrl>

csharp/src/Google.Protobuf.Test/JsonFormatterSettingsTest.cs

@@ -0,0 +1,111 @@
+#region Copyright notice and license
+
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+using Google.Protobuf.Reflection;
+using NUnit.Framework;
+
+// For WrapInQuotes
+
+namespace Google.Protobuf
+{
+    public class JsonFormatterSettingsTest
+    {
+        [Test]
+        public void WithIndentation()
+        {
+            var settings = JsonFormatter.Settings.Default.WithIndentation("\t");
+            Assert.AreEqual("\t", settings.Indentation);
+        }
+
+        [Test]
+        public void WithTypeRegistry()
+        {
+            var typeRegistry = TypeRegistry.Empty;
+            var settings = JsonFormatter.Settings.Default.WithTypeRegistry(typeRegistry);
+            Assert.AreEqual(typeRegistry, settings.TypeRegistry);
+        }
+
+        [Test]
+        public void WithFormatDefaultValues()
+        {
+            var settingsWith = JsonFormatter.Settings.Default.WithFormatDefaultValues(true);
+            Assert.AreEqual(true, settingsWith.FormatDefaultValues);
+
+            var settingsWithout = JsonFormatter.Settings.Default.WithFormatDefaultValues(false);
+            Assert.AreEqual(false, settingsWithout.FormatDefaultValues);
+        }
+
+        [Test]
+        public void WithFormatEnumsAsIntegers()
+        {
+            var settingsWith = JsonFormatter.Settings.Default.WithFormatEnumsAsIntegers(true);
+            Assert.AreEqual(true, settingsWith.FormatEnumsAsIntegers);
+
+            var settingsWithout = JsonFormatter.Settings.Default.WithFormatEnumsAsIntegers(false);
+            Assert.AreEqual(false, settingsWithout.FormatEnumsAsIntegers);
+        }
+
+        [Test]
+        public void WithMethodsPreserveExistingSettings()
+        {
+            var typeRegistry = TypeRegistry.Empty;
+            var baseSettings = JsonFormatter.Settings.Default
+                .WithIndentation("\t")
+                .WithFormatDefaultValues(true)
+                .WithFormatEnumsAsIntegers(true)
+                .WithTypeRegistry(typeRegistry)
+                .WithPreserveProtoFieldNames(true);
+
+            var settings1 = baseSettings.WithIndentation("\t");
+            var settings2 = baseSettings.WithFormatDefaultValues(true);
+            var settings3 = baseSettings.WithFormatEnumsAsIntegers(true);
+            var settings4 = baseSettings.WithTypeRegistry(typeRegistry);
+            var settings5 = baseSettings.WithPreserveProtoFieldNames(true);
+
+            AssertAreEqual(baseSettings, settings1);
+            AssertAreEqual(baseSettings, settings2);
+            AssertAreEqual(baseSettings, settings3);
+            AssertAreEqual(baseSettings, settings4);
+            AssertAreEqual(baseSettings, settings5);
+        }
+
+        private static void AssertAreEqual(JsonFormatter.Settings settings, JsonFormatter.Settings other)
+        {
+            Assert.AreEqual(settings.Indentation, other.Indentation);
+            Assert.AreEqual(settings.FormatDefaultValues, other.FormatDefaultValues);
+            Assert.AreEqual(settings.FormatEnumsAsIntegers, other.FormatEnumsAsIntegers);
+            Assert.AreEqual(settings.TypeRegistry, other.TypeRegistry);
+        }
+    }
+}

csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs

@@ -674,6 +674,200 @@ namespace Google.Protobuf
             AssertWriteValue(value, "[ 1, 2, 3 ]");
         }
 
+        [Test]
+        public void WriteValueWithIndentation_EmptyMessage()
+        {
+            var value = new TestEmptyMessage();
+
+            AssertWriteValue(value, "{}", JsonFormatter.Settings.Default.WithIndentation());
+        }
+
+        [Test]
+        public void WriteValueWithIndentation_NestedTestAllTypes()
+        {
+            var value = new NestedTestAllTypes
+            {
+                Payload = new TestAllTypes
+                {
+                    SingleBool = true,
+                    SingleInt32 = 100,
+                    SingleString = "multiple fields",
+                    RepeatedString = { "string1", "string2" },
+                },
+                Child = new NestedTestAllTypes
+                {
+                    Payload = new TestAllTypes
+                    {
+                        SingleString = "single field",
+                    },
+                },
+                RepeatedChild =
+                {
+                    new NestedTestAllTypes { Payload = new TestAllTypes { SingleString = "child 1", RepeatedString = { "string" } } },
+                    new NestedTestAllTypes { Payload = new TestAllTypes { SingleString = "child 2" } },
+                },
+            };
+
+            const string expectedJson = @"
+{
+  'child': {
+    'payload': {
+      'singleString': 'single field'
+    }
+  },
+  'payload': {
+    'singleInt32': 100,
+    'singleBool': true,
+    'singleString': 'multiple fields',
+    'repeatedString': [
+      'string1',
+      'string2'
+    ]
+  },
+  'repeatedChild': [
+    {
+      'payload': {
+        'singleString': 'child 1',
+        'repeatedString': [
+          'string'
+        ]
+      }
+    },
+    {
+      'payload': {
+        'singleString': 'child 2'
+      }
+    }
+  ]
+}";
+            AssertWriteValue(value, expectedJson, JsonFormatter.Settings.Default.WithIndentation());
+        }
+
+        [Test]
+        public void WriteValueWithIndentation_WellKnownTypes()
+        {
+            var value = new TestWellKnownTypes
+            {
+                StructField = new Struct
+                {
+                    Fields =
+                    {
+                        { "string", Value.ForString("foo") },
+                        { "numbers", Value.ForList(Value.ForNumber(1), Value.ForNumber(2), Value.ForNumber(3)) },
+                        { "emptyList", Value.ForList() },
+                        { "emptyStruct", Value.ForStruct(new Struct()) },
+                    },
+                },
+            };
+
+            const string expectedJson = @"
+{
+  'structField': {
+    'string': 'foo',
+    'numbers': [
+      1,
+      2,
+      3
+    ],
+    'emptyList': [],
+    'emptyStruct': {}
+  }
+}";
+            AssertWriteValue(value, expectedJson, JsonFormatter.Settings.Default.WithIndentation());
+        }
+
+        [Test]
+        public void WriteValueWithIndentation_StructSingleField()
+        {
+            var value = new Struct { Fields = { { "structField1", Value.ForString("structFieldValue1") } } };
+
+            const string expectedJson = @"
+{
+  'structField1': 'structFieldValue1'
+}";
+            AssertWriteValue(value, expectedJson, JsonFormatter.Settings.Default.WithIndentation());
+        }
+
+        [Test]
+        public void WriteValueWithIndentation_StructMultipleFields()
+        {
+            var value = new Struct
+            {
+                Fields =
+                {
+                    { "structField1", Value.ForString("structFieldValue1") },
+                    { "structField2", Value.ForString("structFieldValue2") },
+                    { "structField3", Value.ForString("structFieldValue3") },
+                },
+            };
+
+            const string expectedJson = @"
+{
+  'structField1': 'structFieldValue1',
+  'structField2': 'structFieldValue2',
+  'structField3': 'structFieldValue3'
+}";
+            AssertWriteValue(value, expectedJson, JsonFormatter.Settings.Default.WithIndentation());
+        }
+
+        [Test]
+        public void FormatWithIndentation_EmbeddedMessage()
+        {
+            var value = new TestAllTypes { SingleInt32 = 100, SingleInt64 = 3210987654321L };
+            var formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithIndentation());
+            var valueJson = formatter.Format(value, indentationLevel: 1);
+
+            var actualJson = $@"
+{{
+  ""data"": {valueJson}
+}}";
+            const string expectedJson = @"
+{
+  'data': {
+    'singleInt32': 100,
+    'singleInt64': '3210987654321'
+  }
+}";
+            AssertJson(expectedJson, actualJson.Trim());
+        }
+
+        [Test]
+        public void WriteValueWithIndentation_Map()
+        {
+            var value = new TestMap
+            {
+                MapStringString =
+                {
+                    { "key1", "value1" },
+                    { "key2", "value2" },
+                },
+            };
+
+            const string expectedJson = @"
+{
+  'mapStringString': {
+    'key1': 'value1',
+    'key2': 'value2'
+  }
+}";
+
+            AssertWriteValue(value, expectedJson, JsonFormatter.Settings.Default.WithIndentation());
+        }
+
+        [Test]
+        public void WriteValueWithIndentation_List()
+        {
+            var value = new RepeatedField<int> { 1, 2, 3 };
+            AssertWriteValue(value, "[\n  1,\n  2,\n  3\n]", JsonFormatter.Settings.Default.WithIndentation());
+        }
+
+        [Test]
+        public void WriteValueWithIndentation_CustomIndentation()
+        {
+            var value = new RepeatedField<int> { 1, 2, 3 };
+            AssertWriteValue(value, "[\n\t1,\n\t2,\n\t3\n]", JsonFormatter.Settings.Default.WithIndentation("\t"));
+        }
+
         [Test]
         public void Proto2_DefaultValuesWritten()
         {
@@ -683,7 +877,7 @@ namespace Google.Protobuf
 
         private static void AssertWriteValue(object value, string expectedJson, JsonFormatter.Settings settings = null)
         {
-            var writer = new StringWriter();
+            var writer = new StringWriter { NewLine = "\n" };
             new JsonFormatter(settings ?? JsonFormatter.Settings.Default).WriteValue(writer, value);
             string actual = writer.ToString();
             AssertJson(expectedJson, actual);
@@ -691,13 +885,17 @@ namespace Google.Protobuf
 
         /// <summary>
         /// Checks that the actual JSON is the same as the expected JSON - but after replacing
-        /// all apostrophes in the expected JSON with double quotes. This basically makes the tests easier
-        /// to read.
+        /// all apostrophes in the expected JSON with double quotes, trimming leading whitespace and normalizing new lines.
+        /// This basically makes the tests easier to read.
         /// </summary>
+        /// <remarks>
+        /// Line endings are normalized because indented JSON strings are generated with system-specific line endings,
+        /// while line endings in the test cases are hard-coded, but may be converted during source checkout, depending
+        /// on git settings, causing unpredictability in the test results otherwise.</remarks>
         private static void AssertJson(string expectedJsonWithApostrophes, string actualJson)
         {
-            var expectedJson = expectedJsonWithApostrophes.Replace("'", "\"");
-            Assert.AreEqual(expectedJson, actualJson);
+            var expectedJson = expectedJsonWithApostrophes.Replace("'", "\"").Replace("\r\n", "\n").TrimStart();
+            Assert.AreEqual(expectedJson, actualJson.Replace("\r\n", "\n"));
         }
     }
 }

csharp/src/Google.Protobuf/Google.Protobuf.csproj

@@ -4,7 +4,7 @@
     <Description>C# runtime library for Protocol Buffers - Google's data interchange format.</Description>
     <Copyright>Copyright 2015, Google Inc.</Copyright>
     <AssemblyTitle>Google Protocol Buffers</AssemblyTitle>
-    <VersionPrefix>3.21.3</VersionPrefix>
+    <VersionPrefix>3.21.4</VersionPrefix>
     <LangVersion>10.0</LangVersion>
     <Authors>Google Inc.</Authors>
     <TargetFrameworks>netstandard1.1;netstandard2.0;net45;net50</TargetFrameworks>

csharp/src/Google.Protobuf/JsonFormatter.cs

@@ -63,7 +63,12 @@ namespace Google.Protobuf
         internal const string AnyDiagnosticValueField = "@value";
         internal const string AnyWellKnownTypeValueField = "value";
         private const string NameValueSeparator = ": ";
-        private const string PropertySeparator = ", ";
+        private const string ValueSeparator = ", ";
+        private const string MultilineValueSeparator = ",";
+        private const char ObjectOpenBracket = '{';
+        private const char ObjectCloseBracket = '}';
+        private const char ListBracketOpen = '[';
+        private const char ListBracketClose = ']';
 
         /// <summary>
         /// Returns a formatter using the default settings.
@@ -140,11 +145,26 @@ namespace Google.Protobuf
         /// Formats the specified message as JSON.
         /// </summary>
         /// <param name="message">The message to format.</param>
+        /// <remarks>This method delegates to <c>Format(IMessage, int)</c> with <c>indentationLevel = 0</c>.</remarks>
         /// <returns>The formatted message.</returns>
-        public string Format(IMessage message)
+        public string Format(IMessage message) => Format(message, indentationLevel: 0);
+
+        /// <summary>
+        /// Formats the specified message as JSON.
+        /// </summary>
+        /// <param name="message">The message to format.</param>
+        /// <param name="indentationLevel">Indentation level to start at.</param>
+        /// <remarks>To keep consistent indentation when embedding a message inside another JSON string, set <see cref="indentationLevel"/>. E.g:
+        /// <code>
+        /// var response = $@"{{
+        ///   ""data"": { Format(message, indentationLevel: 1) }
+        /// }}"</code>
+        /// </remarks>
+        /// <returns>The formatted message.</returns>
+        public string Format(IMessage message, int indentationLevel)
         {
             var writer = new StringWriter();
-            Format(message, writer);
+            Format(message, writer, indentationLevel);
             return writer.ToString();
         }
 
@@ -153,19 +173,29 @@ namespace Google.Protobuf
         /// </summary>
         /// <param name="message">The message to format.</param>
         /// <param name="writer">The TextWriter to write the formatted message to.</param>
+        /// <remarks>This method delegates to <c>Format(IMessage, TextWriter, int)</c> with <c>indentationLevel = 0</c>.</remarks>
         /// <returns>The formatted message.</returns>
-        public void Format(IMessage message, TextWriter writer)
+        public void Format(IMessage message, TextWriter writer) => Format(message, writer, indentationLevel: 0);
+
+        /// <summary>
+        /// Formats the specified message as JSON. When <see cref="Settings.Indentation"/> is not null, start indenting at the specified <see cref="indentationLevel"/>.
+        /// </summary>
+        /// <param name="message">The message to format.</param>
+        /// <param name="writer">The TextWriter to write the formatted message to.</param>
+        /// <param name="indentationLevel">Indentation level to start at.</param>
+        /// <remarks>To keep consistent indentation when embedding a message inside another JSON string, set <see cref="indentationLevel"/>.</remarks>
+        public void Format(IMessage message, TextWriter writer, int indentationLevel)
         {
             ProtoPreconditions.CheckNotNull(message, nameof(message));
             ProtoPreconditions.CheckNotNull(writer, nameof(writer));
 
             if (message.Descriptor.IsWellKnownType)
             {
-                WriteWellKnownTypeValue(writer, message.Descriptor, message);
+                WriteWellKnownTypeValue(writer, message.Descriptor, message, indentationLevel);
             }
             else
             {
-                WriteMessage(writer, message);
+                WriteMessage(writer, message, indentationLevel);
             }
         }
 
@@ -192,7 +222,7 @@ namespace Google.Protobuf
             return diagnosticFormatter.Format(message);
         }
 
-        private void WriteMessage(TextWriter writer, IMessage message)
+        private void WriteMessage(TextWriter writer, IMessage message, int indentationLevel)
         {
             if (message == null)
             {
@@ -207,12 +237,13 @@ namespace Google.Protobuf
                     return;
                 }
             }
-            writer.Write("{ ");
-            bool writtenFields = WriteMessageFields(writer, message, false);
-            writer.Write(writtenFields ? " }" : "}");
+
+            WriteBracketOpen(writer, ObjectOpenBracket);
+            bool writtenFields = WriteMessageFields(writer, message, false, indentationLevel + 1);
+            WriteBracketClose(writer, ObjectCloseBracket, writtenFields, indentationLevel);
         }
 
-        private bool WriteMessageFields(TextWriter writer, IMessage message, bool assumeFirstFieldWritten)
+        private bool WriteMessageFields(TextWriter writer, IMessage message, bool assumeFirstFieldWritten, int indentationLevel)
         {
             var fields = message.Descriptor.Fields;
             bool first = !assumeFirstFieldWritten;
@@ -226,10 +257,8 @@ namespace Google.Protobuf
                     continue;
                 }
 
-                if (!first)
-                {
-                    writer.Write(PropertySeparator);
-                }
+                MaybeWriteValueSeparator(writer, first);
+                MaybeWriteValueWhitespace(writer, indentationLevel);
 
                 if (settings.PreserveProtoFieldNames)
                 {
@@ -240,13 +269,23 @@ namespace Google.Protobuf
                     WriteString(writer, accessor.Descriptor.JsonName);
                 }
                 writer.Write(NameValueSeparator);
-                WriteValue(writer, value);
+                WriteValue(writer, value, indentationLevel);
 
                 first = false;
             }
             return !first;
         }
 
+        private void MaybeWriteValueSeparator(TextWriter writer, bool first)
+        {
+            if (first)
+            {
+                return;
+            }
+
+            writer.Write(settings.Indentation == null ? ValueSeparator : MultilineValueSeparator);
+        }
+
         /// <summary>
         /// Determines whether or not a field value should be serialized according to the field,
         /// its value in the message, and the settings of this formatter.
@@ -342,7 +381,19 @@ namespace Google.Protobuf
         /// </summary>
         /// <param name="writer">The writer to write the value to. Must not be null.</param>
         /// <param name="value">The value to write. May be null.</param>
-        public void WriteValue(TextWriter writer, object value)
+        /// <remarks>Delegates to <c>WriteValue(TextWriter, object, int)</c> with <c>indentationLevel = 0</c>.</remarks>
+        public void WriteValue(TextWriter writer, object value) => WriteValue(writer, value, 0);
+
+        /// <summary>
+        /// Writes a single value to the given writer as JSON. Only types understood by
+        /// Protocol Buffers can be written in this way. This method is only exposed for
+        /// advanced use cases; most users should be using <see cref="Format(IMessage)"/>
+        /// or <see cref="Format(IMessage, TextWriter)"/>.
+        /// </summary>
+        /// <param name="writer">The writer to write the value to. Must not be null.</param>
+        /// <param name="value">The value to write. May be null.</param>
+        /// <param name="indentationLevel">The current indentationLevel. Not used when <see cref="Settings.Indentation"/> is null.</param>
+        public void WriteValue(TextWriter writer, object value, int indentationLevel)
         {
             if (value == null || value is NullValue)
             {
@@ -365,11 +416,11 @@ namespace Google.Protobuf
             }
             else if (value is IDictionary dictionary)
             {
-                WriteDictionary(writer, dictionary);
+                WriteDictionary(writer, dictionary, indentationLevel);
             }
             else if (value is IList list)
             {
-                WriteList(writer, list);
+                WriteList(writer, list, indentationLevel);
             }
             else if (value is int || value is uint)
             {
@@ -418,7 +469,7 @@ namespace Google.Protobuf
             }
             else if (value is IMessage message)
             {
-                Format(message, writer);
+                Format(message, writer, indentationLevel);
             }
             else
             {
@@ -432,7 +483,7 @@ namespace Google.Protobuf
         /// values are using the embedded well-known types, in order to allow for dynamic messages
         /// in the future.
         /// </summary>
-        private void WriteWellKnownTypeValue(TextWriter writer, MessageDescriptor descriptor, object value)
+        private void WriteWellKnownTypeValue(TextWriter writer, MessageDescriptor descriptor, object value, int indentationLevel)
         {
             // Currently, we can never actually get here, because null values are always handled by the caller. But if we *could*,
             // this would do the right thing.
@@ -472,26 +523,26 @@ namespace Google.Protobuf
             }
             if (descriptor.FullName == Struct.Descriptor.FullName)
             {
-                WriteStruct(writer, (IMessage)value);
+                WriteStruct(writer, (IMessage)value, indentationLevel);
                 return;
             }
             if (descriptor.FullName == ListValue.Descriptor.FullName)
             {
                 var fieldAccessor = descriptor.Fields[ListValue.ValuesFieldNumber].Accessor;
-                WriteList(writer, (IList)fieldAccessor.GetValue((IMessage)value));
+                WriteList(writer, (IList)fieldAccessor.GetValue((IMessage)value), indentationLevel);
                 return;
             }
             if (descriptor.FullName == Value.Descriptor.FullName)
             {
-                WriteStructFieldValue(writer, (IMessage)value);
+                WriteStructFieldValue(writer, (IMessage)value, indentationLevel);
                 return;
             }
             if (descriptor.FullName == Any.Descriptor.FullName)
             {
-                WriteAny(writer, (IMessage)value);
+                WriteAny(writer, (IMessage)value, indentationLevel);
                 return;
             }
-            WriteMessage(writer, (IMessage)value);
+            WriteMessage(writer, (IMessage)value, indentationLevel);
         }
 
         private void WriteTimestamp(TextWriter writer, IMessage value)
@@ -519,7 +570,7 @@ namespace Google.Protobuf
             writer.Write(FieldMask.ToJson(paths, DiagnosticOnly));
         }
 
-        private void WriteAny(TextWriter writer, IMessage value)
+        private void WriteAny(TextWriter writer, IMessage value, int indentationLevel)
         {
             if (DiagnosticOnly)
             {
@@ -536,23 +587,23 @@ namespace Google.Protobuf
                 throw new InvalidOperationException($"Type registry has no descriptor for type name '{typeName}'");
             }
             IMessage message = descriptor.Parser.ParseFrom(data);
-            writer.Write("{ ");
+            WriteBracketOpen(writer, ObjectOpenBracket);
             WriteString(writer, AnyTypeUrlField);
             writer.Write(NameValueSeparator);
             WriteString(writer, typeUrl);
 
             if (descriptor.IsWellKnownType)
             {
-                writer.Write(PropertySeparator);
+                writer.Write(ValueSeparator);
                 WriteString(writer, AnyWellKnownTypeValueField);
                 writer.Write(NameValueSeparator);
-                WriteWellKnownTypeValue(writer, descriptor, message);
+                WriteWellKnownTypeValue(writer, descriptor, message, indentationLevel);
             }
             else
             {
-                WriteMessageFields(writer, message, true);
+                WriteMessageFields(writer, message, true, indentationLevel);
             }
-            writer.Write(" }");
+            WriteBracketClose(writer, ObjectCloseBracket, true, indentationLevel);
         }
 
         private void WriteDiagnosticOnlyAny(TextWriter writer, IMessage value)
@@ -563,7 +614,7 @@ namespace Google.Protobuf
             WriteString(writer, AnyTypeUrlField);
             writer.Write(NameValueSeparator);
             WriteString(writer, typeUrl);
-            writer.Write(PropertySeparator);
+            writer.Write(ValueSeparator);
             WriteString(writer, AnyDiagnosticValueField);
             writer.Write(NameValueSeparator);
             writer.Write('"');
@@ -572,9 +623,9 @@ namespace Google.Protobuf
             writer.Write(" }");
         }
 
-        private void WriteStruct(TextWriter writer, IMessage message)
+        private void WriteStruct(TextWriter writer, IMessage message, int indentationLevel)
         {
-            writer.Write("{ ");
+            WriteBracketOpen(writer, ObjectOpenBracket);
             IDictionary fields = (IDictionary) message.Descriptor.Fields[Struct.FieldsFieldNumber].Accessor.GetValue(message);
             bool first = true;
             foreach (DictionaryEntry entry in fields)
@@ -586,19 +637,17 @@ namespace Google.Protobuf
                     throw new InvalidOperationException("Struct fields cannot have an empty key or a null value.");
                 }
 
-                if (!first)
-                {
-                    writer.Write(PropertySeparator);
-                }
+                MaybeWriteValueSeparator(writer, first);
+                MaybeWriteValueWhitespace(writer, indentationLevel + 1);
                 WriteString(writer, key);
                 writer.Write(NameValueSeparator);
-                WriteStructFieldValue(writer, value);
+                WriteStructFieldValue(writer, value, indentationLevel + 1);
                 first = false;
             }
-            writer.Write(first ? "}" : " }");
+            WriteBracketClose(writer, ObjectCloseBracket, !first, indentationLevel);
         }
 
-        private void WriteStructFieldValue(TextWriter writer, IMessage message)
+        private void WriteStructFieldValue(TextWriter writer, IMessage message, int indentationLevel)
         {
             var specifiedField = message.Descriptor.Oneofs[0].Accessor.GetCaseFieldDescriptor(message);
             if (specifiedField == null)
@@ -619,7 +668,7 @@ namespace Google.Protobuf
                 case Value.ListValueFieldNumber:
                     // Structs and ListValues are nested messages, and already well-known types.
                     var nestedMessage = (IMessage) specifiedField.Accessor.GetValue(message);
-                    WriteWellKnownTypeValue(writer, nestedMessage.Descriptor, nestedMessage);
+                    WriteWellKnownTypeValue(writer, nestedMessage.Descriptor, nestedMessage, indentationLevel);
                     return;
                 case Value.NullValueFieldNumber:
                     WriteNull(writer);
@@ -629,33 +678,30 @@ namespace Google.Protobuf
             }
         }
 
-        internal void WriteList(TextWriter writer, IList list)
+        internal void WriteList(TextWriter writer, IList list, int indentationLevel = 0)
         {
-            writer.Write("[ ");
+            WriteBracketOpen(writer, ListBracketOpen);
+
             bool first = true;
             foreach (var value in list)
             {
-                if (!first)
-                {
-                    writer.Write(PropertySeparator);
-                }
-                WriteValue(writer, value);
+                MaybeWriteValueSeparator(writer, first);
+                MaybeWriteValueWhitespace(writer, indentationLevel + 1);
+                WriteValue(writer, value, indentationLevel + 1);
                 first = false;
             }
-            writer.Write(first ? "]" : " ]");
+
+            WriteBracketClose(writer, ListBracketClose, !first, indentationLevel);
         }
 
-        internal void WriteDictionary(TextWriter writer, IDictionary dictionary)
+        internal void WriteDictionary(TextWriter writer, IDictionary dictionary, int indentationLevel = 0)
         {
-            writer.Write("{ ");
+            WriteBracketOpen(writer, ObjectOpenBracket);
+
             bool first = true;
             // This will box each pair. Could use IDictionaryEnumerator, but that's ugly in terms of disposal.
             foreach (DictionaryEntry pair in dictionary)
             {
-                if (!first)
-                {
-                    writer.Write(PropertySeparator);
-                }
                 string keyText;
                 if (pair.Key is string s)
                 {
@@ -677,12 +723,16 @@ namespace Google.Protobuf
                     }
                     throw new ArgumentException("Unhandled dictionary key type: " + pair.Key.GetType());
                 }
+
+                MaybeWriteValueSeparator(writer, first);
+                MaybeWriteValueWhitespace(writer, indentationLevel + 1);
                 WriteString(writer, keyText);
                 writer.Write(NameValueSeparator);
                 WriteValue(writer, pair.Value);
                 first = false;
             }
-            writer.Write(first ? "}" : " }");
+
+            WriteBracketClose(writer, ObjectCloseBracket, !first, indentationLevel);
         }
 
         /// <summary>
@@ -766,6 +816,49 @@ namespace Google.Protobuf
             writer.Write(Hex[(c >> 0) & 0xf]);
         }
 
+        private void WriteBracketOpen(TextWriter writer, char openChar)
+        {
+            writer.Write(openChar);
+            if (settings.Indentation == null)
+            {
+                writer.Write(' ');
+            }
+        }
+
+        private void WriteBracketClose(TextWriter writer, char closeChar, bool hasFields, int indentationLevel)
+        {
+            if (hasFields)
+            {
+                if (settings.Indentation != null)
+                {
+                    writer.WriteLine();
+                    WriteIndentation(writer, indentationLevel);
+                }
+                else
+                {
+                    writer.Write(" ");
+                }
+            }
+
+            writer.Write(closeChar);
+        }
+
+        private void MaybeWriteValueWhitespace(TextWriter writer, int indentationLevel)
+        {
+            if (settings.Indentation != null) {
+                writer.WriteLine();
+                WriteIndentation(writer, indentationLevel);
+            }
+        }
+
+        private void WriteIndentation(TextWriter writer, int indentationLevel)
+        {
+            for (int i = 0; i < indentationLevel; i++)
+            {
+                writer.Write(settings.Indentation);
+            }
+        }
+
         /// <summary>
         /// Settings controlling JSON formatting.
         /// </summary>
@@ -806,6 +899,10 @@ namespace Google.Protobuf
             /// </summary>
             public bool PreserveProtoFieldNames { get; }
 
+            /// <summary>
+            /// Indentation string, used for formatting. Setting null disables indentation.
+            /// </summary>
+            public string Indentation { get; }
 
             /// <summary>
             /// Creates a new <see cref="Settings"/> object with the specified formatting of default values
@@ -833,40 +930,54 @@ namespace Google.Protobuf
             /// <param name="typeRegistry">The <see cref="TypeRegistry"/> to use when formatting <see cref="Any"/> messages. TypeRegistry.Empty will be used if it is null.</param>
             /// <param name="formatEnumsAsIntegers"><c>true</c> to format the enums as integers; <c>false</c> to format enums as enum names.</param>
             /// <param name="preserveProtoFieldNames"><c>true</c> to preserve proto field names; <c>false</c> to convert them to lowerCamelCase.</param>
+            /// <param name="indentation">The indentation string to use for multi-line formatting. <c>null</c> to disable multi-line format.</param>
             private Settings(bool formatDefaultValues,
                             TypeRegistry typeRegistry,
                             bool formatEnumsAsIntegers,
-                            bool preserveProtoFieldNames)
+                            bool preserveProtoFieldNames,
+                            string indentation = null)
             {
                 FormatDefaultValues = formatDefaultValues;
                 TypeRegistry = typeRegistry ?? TypeRegistry.Empty;
                 FormatEnumsAsIntegers = formatEnumsAsIntegers;
                 PreserveProtoFieldNames = preserveProtoFieldNames;
+                Indentation = indentation;
             }
 
             /// <summary>
             /// Creates a new <see cref="Settings"/> object with the specified formatting of default values and the current settings.
             /// </summary>
             /// <param name="formatDefaultValues"><c>true</c> if default values (0, empty strings etc) should be formatted; <c>false</c> otherwise.</param>
-            public Settings WithFormatDefaultValues(bool formatDefaultValues) => new Settings(formatDefaultValues, TypeRegistry, FormatEnumsAsIntegers, PreserveProtoFieldNames);
+            public Settings WithFormatDefaultValues(bool formatDefaultValues) => new Settings(formatDefaultValues, TypeRegistry, FormatEnumsAsIntegers, PreserveProtoFieldNames, Indentation);
 
             /// <summary>
             /// Creates a new <see cref="Settings"/> object with the specified type registry and the current settings.
             /// </summary>
             /// <param name="typeRegistry">The <see cref="TypeRegistry"/> to use when formatting <see cref="Any"/> messages.</param>
-            public Settings WithTypeRegistry(TypeRegistry typeRegistry) => new Settings(FormatDefaultValues, typeRegistry, FormatEnumsAsIntegers, PreserveProtoFieldNames);
+            public Settings WithTypeRegistry(TypeRegistry typeRegistry) => new Settings(FormatDefaultValues, typeRegistry, FormatEnumsAsIntegers, PreserveProtoFieldNames, Indentation);
 
             /// <summary>
             /// Creates a new <see cref="Settings"/> object with the specified enums formatting option and the current settings.
             /// </summary>
             /// <param name="formatEnumsAsIntegers"><c>true</c> to format the enums as integers; <c>false</c> to format enums as enum names.</param>
-            public Settings WithFormatEnumsAsIntegers(bool formatEnumsAsIntegers) => new Settings(FormatDefaultValues, TypeRegistry, formatEnumsAsIntegers, PreserveProtoFieldNames);
+            public Settings WithFormatEnumsAsIntegers(bool formatEnumsAsIntegers) => new Settings(FormatDefaultValues, TypeRegistry, formatEnumsAsIntegers, PreserveProtoFieldNames, Indentation);
 
             /// <summary>
             /// Creates a new <see cref="Settings"/> object with the specified field name formatting option and the current settings.
             /// </summary>
             /// <param name="preserveProtoFieldNames"><c>true</c> to preserve proto field names; <c>false</c> to convert them to lowerCamelCase.</param>
-            public Settings WithPreserveProtoFieldNames(bool preserveProtoFieldNames) => new Settings(FormatDefaultValues, TypeRegistry, FormatEnumsAsIntegers, preserveProtoFieldNames);
+            public Settings WithPreserveProtoFieldNames(bool preserveProtoFieldNames) => new Settings(FormatDefaultValues, TypeRegistry, FormatEnumsAsIntegers, preserveProtoFieldNames, Indentation);
+
+            /// <summary>
+            /// Creates a new <see cref="Settings"/> object with the specified indentation and the current settings.
+            /// </summary>
+            /// <param name="indentation">The string to output for each level of indentation (nesting). The default is two spaces per level. Use null to disable indentation entirely.</param>
+            /// <remarks>A non-null value for <see cref="Indentation"/> will insert additional line-breaks to the JSON output.
+            /// Each line will contain either a single value, or braces. The default line-break is determined by <see cref="Environment.NewLine"/>,
+            /// which is <c>"\n"</c> on Unix platforms, and <c>"\r\n"</c> on Windows. If <see cref="JsonFormatter"/> seems to produce empty lines,
+            /// you need to pass a <see cref="TextWriter"/> that uses a <c>"\n"</c> newline. See <see cref="JsonFormatter.Format(Google.Protobuf.IMessage, TextWriter)"/>.
+            /// </remarks>
+            public Settings WithIndentation(string indentation = "  ") => new Settings(FormatDefaultValues, TypeRegistry, FormatEnumsAsIntegers, PreserveProtoFieldNames, indentation);
         }
 
         // Effectively a cache of mapping from enum values to the original name as specified in the proto file,

java/README.md

@@ -23,7 +23,7 @@ If you are using Maven, use the following:
 <dependency>
   <groupId>com.google.protobuf</groupId>
   <artifactId>protobuf-java</artifactId>
-  <version>3.21.3</version>
+  <version>3.21.4</version>
 </dependency>
 ```
 
@@ -37,7 +37,7 @@ protobuf-java-util package:
 <dependency>
   <groupId>com.google.protobuf</groupId>
   <artifactId>protobuf-java-util</artifactId>
-  <version>3.21.3</version>
+  <version>3.21.4</version>
 </dependency>
 ```
 
@@ -45,7 +45,7 @@ protobuf-java-util package:
 
 If you are using Gradle, add the following to your `build.gradle` file's dependencies:
 ```
-    implementation 'com.google.protobuf:protobuf-java:3.21.3'
+    implementation 'com.google.protobuf:protobuf-java:3.21.4'
 ```
 Again, be sure to check that the version number matches (or is newer than) the version number of protoc that you are using.
 

java/bom/pom.xml

@@ -4,7 +4,7 @@
 
   <groupId>com.google.protobuf</groupId>
   <artifactId>protobuf-bom</artifactId>
-  <version>3.21.3</version>
+  <version>3.21.4</version>
   <packaging>pom</packaging>
 
   <name>Protocol Buffers [BOM]</name>

java/core/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.google.protobuf</groupId>
     <artifactId>protobuf-parent</artifactId>
-    <version>3.21.3</version>
+    <version>3.21.4</version>
   </parent>
 
   <artifactId>protobuf-java</artifactId>

java/kotlin-lite/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.google.protobuf</groupId>
     <artifactId>protobuf-parent</artifactId>
-    <version>3.21.3</version>
+    <version>3.21.4</version>
   </parent>
 
   <artifactId>protobuf-kotlin-lite</artifactId>

java/kotlin/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.google.protobuf</groupId>
     <artifactId>protobuf-parent</artifactId>
-    <version>3.21.3</version>
+    <version>3.21.4</version>
   </parent>
 
   <artifactId>protobuf-kotlin</artifactId>

java/lite.md

@@ -29,7 +29,7 @@ protobuf Java Lite runtime. If you are using Maven, include the following:
 <dependency>
   <groupId>com.google.protobuf</groupId>
   <artifactId>protobuf-javalite</artifactId>
-  <version>3.21.3</version>
+  <version>3.21.4</version>
 </dependency>
 ```
 

java/lite/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.google.protobuf</groupId>
     <artifactId>protobuf-parent</artifactId>
-    <version>3.21.3</version>
+    <version>3.21.4</version>
   </parent>
 
   <artifactId>protobuf-javalite</artifactId>

java/pom.xml

@@ -4,7 +4,7 @@
 
   <groupId>com.google.protobuf</groupId>
   <artifactId>protobuf-parent</artifactId>
-  <version>3.21.3</version>
+  <version>3.21.4</version>
   <packaging>pom</packaging>
 
   <name>Protocol Buffers [Parent]</name>

java/util/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.google.protobuf</groupId>
     <artifactId>protobuf-parent</artifactId>
-    <version>3.21.3</version>
+    <version>3.21.4</version>
   </parent>
 
   <artifactId>protobuf-java-util</artifactId>

kokoro/linux/cmake/build.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Build file to set up and run tests based on distribution archive
+
+set -eux
+
+# Change to repo root
+cd $(dirname $0)/../../..
+GIT_REPO_ROOT=`pwd`
+
+CONTAINER_IMAGE=gcr.io/protobuf-build/cmake/linux@sha256:79e6ed9d7f3f8e56167a3309a521e5b7e6a212bfb19855c65ee1cbb6f9099671
+
+# Update git submodules
+git submodule update --init --recursive
+
+tmpfile=$(mktemp -u)
+
+docker run \
+  --cidfile $tmpfile \
+  -v $GIT_REPO_ROOT:/workspace \
+  $CONTAINER_IMAGE \
+  /test.sh -Dprotobuf_BUILD_CONFORMANCE=ON
+
+# Save logs for Kokoro
+docker cp \
+  `cat $tmpfile`:/workspace/logs $KOKORO_ARTIFACTS_DIR

kokoro/linux/cmake/continuous.cfg

@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/cmake/build.sh"
+timeout_mins: 1440
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

kokoro/linux/cmake/presubmit.cfg

@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/cmake/build.sh"
+timeout_mins: 1440
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

kokoro/linux/cmake_install/build.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Build file to set up and run tests based on distribution archive
+
+set -eux
+
+# Change to repo root
+cd $(dirname $0)/../../..
+GIT_REPO_ROOT=`pwd`
+
+CONTAINER_IMAGE=gcr.io/protobuf-build/cmake/linux@sha256:79e6ed9d7f3f8e56167a3309a521e5b7e6a212bfb19855c65ee1cbb6f9099671
+
+# Update git submodules
+git submodule update --init --recursive
+
+tmpfile=$(mktemp -u)
+
+docker run \
+  --cidfile $tmpfile \
+  -v $GIT_REPO_ROOT:/workspace \
+  $CONTAINER_IMAGE \
+  "/install.sh && /test.sh \
+  -Dprotobuf_REMOVE_INSTALLED_HEADERS=ON \
+  -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF \
+  -Dprotobuf_BUILD_CONFORMANCE=ON"
+
+
+# Save logs for Kokoro
+docker cp \
+  `cat $tmpfile`:/workspace/logs $KOKORO_ARTIFACTS_DIR

kokoro/linux/cmake_install/continuous.cfg

@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/cmake_install/build.sh"
+timeout_mins: 1440
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

kokoro/linux/cmake_install/presubmit.cfg

@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/cmake_install/build.sh"
+timeout_mins: 1440
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

kokoro/linux/cmake_ninja/build.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Build file to set up and run tests based on distribution archive
+
+set -eux
+
+# Change to repo root
+cd $(dirname $0)/../../..
+GIT_REPO_ROOT=`pwd`
+
+CONTAINER_IMAGE=gcr.io/protobuf-build/cmake/linux@sha256:79e6ed9d7f3f8e56167a3309a521e5b7e6a212bfb19855c65ee1cbb6f9099671
+
+# Update git submodules
+git submodule update --init --recursive
+
+tmpfile=$(mktemp -u)
+
+docker run \
+  --cidfile $tmpfile \
+  -v $GIT_REPO_ROOT:/workspace \
+  $CONTAINER_IMAGE \
+  /test.sh -G Ninja -Dprotobuf_BUILD_CONFORMANCE=ON
+
+# Save logs for Kokoro
+docker cp \
+  `cat $tmpfile`:/workspace/logs $KOKORO_ARTIFACTS_DIR

kokoro/linux/cmake_ninja/continuous.cfg

@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/cmake_ninja/build.sh"
+timeout_mins: 1440
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

kokoro/linux/cmake_ninja/presubmit.cfg

@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/cmake_ninja/build.sh"
+timeout_mins: 1440
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.*"
+  }
+}

kokoro/windows/cmake/build.bat

@@ -0,0 +1,4 @@
+@rem enter repo root
+cd /d %~dp0\..\..\..
+
+@rem TODO(mkruskal) Implement tests

kokoro/windows/cmake/continuous.cfg

@@ -0,0 +1,5 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/windows/cmake/build.bat"
+timeout_mins: 1440

kokoro/windows/cmake/presubmit.cfg

@@ -0,0 +1,5 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/windows/cmake/build.bat"
+timeout_mins: 1440

kokoro/windows/cmake_install/build.bat

@@ -0,0 +1,4 @@
+@rem enter repo root
+cd /d %~dp0\..\..\..
+
+@rem TODO(mkruskal) Implement tests

kokoro/windows/cmake_install/continuous.cfg

@@ -0,0 +1,5 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/windows/cmake_install/build.bat"
+timeout_mins: 1440

kokoro/windows/cmake_install/presubmit.cfg

@@ -0,0 +1,5 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/windows/cmake_install/build.bat"
+timeout_mins: 1440

kokoro/windows/cmake_nmake/build.bat

@@ -0,0 +1,4 @@
+@rem enter repo root
+cd /d %~dp0\..\..\..
+
+@rem TODO(mkruskal) Implement tests

kokoro/windows/cmake_nmake/continuous.cfg

@@ -0,0 +1,5 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/windows/cmake_nmake/build.bat"
+timeout_mins: 1440

kokoro/windows/cmake_nmake/presubmit.cfg

@@ -0,0 +1,5 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/windows/cmake_nmake/build.bat"
+timeout_mins: 1440

php/ext/google/protobuf/package.xml

@@ -10,11 +10,11 @@
   <email>protobuf-packages@google.com</email>
   <active>yes</active>
  </lead>
- <date>2022-07-21</date>
- <time>10:19:47</time>
+ <date>2022-07-25</date>
+ <time>13:21:48</time>
  <version>
-  <release>3.21.3</release>
-  <api>3.21.3</api>
+  <release>3.21.4</release>
+  <api>3.21.4</api>
  </version>
  <stability>
   <release>stable</release>
@@ -1373,5 +1373,20 @@ G  A release.
    <notes>
    </notes>
   </release>
+  <release>
+   <version>
+    <release>3.21.4</release>
+    <api>3.21.4</api>
+   </version>
+   <stability>
+    <release>stable</release>
+    <api>stable</api>
+   </stability>
+   <date>2022-07-25</date>
+   <time>13:21:48</time>
+   <license uri="https://opensource.org/licenses/BSD-3-Clause">BSD-3-Clause</license>
+   <notes>
+   </notes>
+  </release>
  </changelog>
 </package>

php/ext/google/protobuf/protobuf.h

@@ -127,7 +127,7 @@ ZEND_BEGIN_ARG_INFO_EX(arginfo_setter, 0, 0, 1)
   ZEND_ARG_INFO(0, value)
 ZEND_END_ARG_INFO()
 
-#define PHP_PROTOBUF_VERSION "3.21.3"
+#define PHP_PROTOBUF_VERSION "3.21.4"
 
 // ptr -> PHP object cache. This is a weak map that caches lazily-created
 // wrapper objects around upb types:

protobuf_version.bzl

@@ -1,3 +1,3 @@
-PROTOC_VERSION = '21.3'
-PROTOBUF_JAVA_VERSION = '3.21.3'
-PROTOBUF_PYTHON_VERSION = '4.21.3'
+PROTOC_VERSION = '21.4'
+PROTOBUF_JAVA_VERSION = '3.21.4'
+PROTOBUF_PYTHON_VERSION = '4.21.4'

protoc-artifacts/pom.xml

@@ -8,7 +8,7 @@
   </parent>
   <groupId>com.google.protobuf</groupId>
   <artifactId>protoc</artifactId>
-  <version>3.21.3</version>
+  <version>3.21.4</version>
   <packaging>pom</packaging>
   <name>Protobuf Compiler</name>
   <description>

python/release.sh

@@ -19,11 +19,24 @@ function run_install_test() {
   chmod +x test-venv/bin/protoc
 
   source test-venv/bin/activate
-  pip install -i ${PYPI} protobuf==${VERSION} --no-cache-dir
+  (pip install -i ${PYPI} protobuf==${VERSION} --no-cache-dir) || (retry_pip_install ${PYPI} ${VERSION})
   deactivate
   rm -fr test-venv
 }
 
+function retry_pip_install() {
+  local PYPI=$1
+  local VERSION=$2
+  
+  read -p "pip install failed, possibly due to delay between upload and availability on pip. Retry? [y/n]" -r
+  echo
+  if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    exit 1
+  fi
+
+  (pip install -i ${PYPI} protobuf==${VERSION} --no-cache-dir) || (retry_pip_install ${PYPI} ${VERSION})
+}
+
 
 [ $# -lt 1 ] && {
   echo "Usage: $0 VERSION ["
@@ -86,13 +99,16 @@ python3 setup.py test
 python3 setup.py sdist
 twine upload --skip-existing -r testpypi -u protobuf-wheel-test dist/*
 
-# Test locally with different python versions.
+# Sleep to allow time for distribution to be available on pip.
+sleep 5m
+
+# Test locally.
 run_install_test ${TESTING_VERSION} python3 https://test.pypi.org/simple
 
 # Deploy egg/wheel packages to testing PyPI and test again.
 python3 setup.py clean build bdist_wheel
 twine upload --skip-existing -r testpypi -u protobuf-wheel-test dist/*
-
+sleep 5m
 run_install_test ${TESTING_VERSION} python3 https://test.pypi.org/simple
 
 echo "All install tests have passed using testing PyPI."

ruby/google-protobuf.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = "google-protobuf"
-  s.version     = "3.21.3"
+  s.version     = "3.21.4"
   git_tag       = "v#{s.version.to_s.sub('.rc.', '-rc')}" # Converts X.Y.Z.rc.N to vX.Y.Z-rcN, used for the git tag
   s.licenses    = ["BSD-3-Clause"]
   s.summary     = "Protocol Buffers"

ruby/pom.xml

@@ -9,7 +9,7 @@
 
     <groupId>com.google.protobuf.jruby</groupId>
     <artifactId>protobuf-jruby</artifactId>
-    <version>3.21.3</version>
+    <version>3.21.4</version>
     <name>Protocol Buffer JRuby native extension</name>
     <description>
       Protocol Buffers are a way of encoding structured data in an efficient yet
@@ -76,7 +76,7 @@
         <dependency>
           <groupId>com.google.protobuf</groupId>
           <artifactId>protobuf-java-util</artifactId>
-          <version>3.21.3</version>
+          <version>3.21.4</version>
         </dependency>
         <dependency>
             <groupId>org.jruby</groupId>

src/Makefile.am

@@ -774,6 +774,7 @@ protobuf_test_SOURCES =                                        \
   google/protobuf/compiler/csharp/csharp_generator_unittest.cc \
   google/protobuf/compiler/importer_unittest.cc                \
   google/protobuf/compiler/java/doc_comment_unittest.cc        \
+  google/protobuf/compiler/java/message_serialization_unittest.cc \
   google/protobuf/compiler/java/plugin_unittest.cc             \
   google/protobuf/compiler/mock_code_generator.cc              \
   google/protobuf/compiler/mock_code_generator.h               \

src/file_lists.cmake

@@ -771,6 +771,7 @@ set(compiler_test_files
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/csharp/csharp_generator_unittest.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/importer_unittest.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/doc_comment_unittest.cc
+  ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_serialization_unittest.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/plugin_unittest.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/objectivec_helpers_unittest.cc
   ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/parser_unittest.cc

src/google/protobuf/arena_unittest.cc

@@ -1419,7 +1419,7 @@ TEST(ArenaTest, BlockSizeDoubling) {
   ASSERT_GT(arena.SpaceAllocated(), first_block_size);
   auto second_block_size = (arena.SpaceAllocated() - first_block_size);
 
-  EXPECT_EQ(second_block_size, 2*first_block_size);
+  EXPECT_GE(second_block_size, 2*first_block_size);
 }
 
 TEST(ArenaTest, Alignment) {

src/google/protobuf/compiler/command_line_interface.cc

@@ -339,9 +339,12 @@ class CommandLineInterface::ErrorPrinter
   void AddErrorOrWarning(const std::string& filename, int line, int column,
                          const std::string& message, const std::string& type,
                          std::ostream& out) {
-    // Print full path when running under MSVS
     std::string dfile;
-    if (format_ == CommandLineInterface::ERROR_FORMAT_MSVS &&
+    if (
+#ifndef PROTOBUF_OPENSOURCE
+        // Print full path when running under MSVS
+        format_ == CommandLineInterface::ERROR_FORMAT_MSVS &&
+#endif  // !PROTOBUF_OPENSOURCE
         tree_ != nullptr && tree_->VirtualFileToDiskFile(filename, &dfile)) {
       out << dfile;
     } else {

src/google/protobuf/compiler/cpp/field.cc

@@ -330,7 +330,6 @@ void FieldGenerator::GenerateCopyConstructorCode(io::Printer* printer) const {
   }
 }
 
-
 void SetCommonOneofFieldVariables(
     const FieldDescriptor* descriptor,
     std::map<std::string, std::string>* variables) {

src/google/protobuf/compiler/cpp/field.h

@@ -208,7 +208,6 @@ class FieldGenerator {
 
   virtual bool IsInlined() const { return false; }
 
-
   virtual ArenaDtorNeeds NeedsArenaDestructor() const {
     return ArenaDtorNeeds::kNone;
   }

src/google/protobuf/compiler/cpp/file.cc

@@ -495,12 +495,10 @@ void FileGenerator::GenerateSourceDefaultInstance(int idx,
     generator->GenerateInitDefaultSplitInstance(printer);
     format(
         "} {}\n"
-        "  ~$1$() {}\n"
         "  union {\n"
-        "    $2$ _instance;\n"
+        "    $1$ _instance;\n"
         "  };\n"
         "};\n",
-        DefaultInstanceType(generator->descriptor_, options_, /*split=*/true),
         StrCat(generator->classname_, "::Impl_::Split"));
     // NO_DESTROY is not necessary for correctness. The empty destructor is
     // enough. However, the empty destructor fails to be elided in some
@@ -508,7 +506,7 @@ void FileGenerator::GenerateSourceDefaultInstance(int idx,
     // there just to improve performance and binary size in these builds.
     format(
         "PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT "
-        "PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 $1$ $2$;\n",
+        "PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 const $1$ $2$;\n",
         DefaultInstanceType(generator->descriptor_, options_, /*split=*/true),
         DefaultInstanceName(generator->descriptor_, options_, /*split=*/true));
   }
@@ -999,7 +997,7 @@ class FileGenerator::ForwardDeclarations {
       const Descriptor* class_desc = p.second;
       format(
           "struct $1$;\n"
-          "$dllexport_decl $extern $1$ $2$;\n",
+          "$dllexport_decl $extern const $1$ $2$;\n",
           DefaultInstanceType(class_desc, options, /*split=*/true),
           DefaultInstanceName(class_desc, options, /*split=*/true));
     }

src/google/protobuf/compiler/cpp/helpers.cc

@@ -176,7 +176,6 @@ static const char* const kKeywordList[] = {
 #endif  // !PROTOBUF_FUTURE_BREAKING_CHANGES
 };
 
-
 static std::unordered_set<std::string>* MakeKeywordsMap() {
   auto* result = new std::unordered_set<std::string>();
   for (const auto keyword : kKeywordList) {
@@ -525,7 +524,6 @@ std::string FieldName(const FieldDescriptor* field) {
   return result;
 }
 
-
 std::string FieldMemberName(const FieldDescriptor* field, bool split) {
   StringPiece prefix =
       IsMapEntryMessage(field->containing_type()) ? "" : "_impl_.";
@@ -876,8 +874,6 @@ std::string SafeFunctionName(const Descriptor* descriptor,
 bool IsProfileDriven(const Options& options) {
   return options.access_info_map != nullptr;
 }
-
-
 bool IsStringInlined(const FieldDescriptor* descriptor,
                      const Options& options) {
   (void)descriptor;

src/google/protobuf/compiler/cpp/message.cc

@@ -828,7 +828,6 @@ void MessageGenerator::GenerateFieldAccessorDeclarations(io::Printer* printer) {
 
     // Generate type-specific accessor declarations.
     field_generators_.get(field).GenerateAccessorDeclarations(printer);
-
     format("\n");
   }
 
@@ -1238,41 +1237,41 @@ void MessageGenerator::GenerateFieldAccessorDefinitions(io::Printer* printer) {
 
     Formatter::SaveState saver(&format);
     format.AddMap(vars);
-      // Generate has_$name$() or $name$_size().
-      if (field->is_repeated()) {
-        if (IsFieldStripped(field, options_)) {
-          format(
-              "inline int $classname$::$name$_size() const { "
-              "__builtin_trap(); }\n");
-        } else {
-          format(
-              "inline int $classname$::_internal_$name$_size() const {\n"
-              "  return $field$$1$.size();\n"
-              "}\n"
-              "inline int $classname$::$name$_size() const {\n"
-              "$annotate_size$"
-              "  return _internal_$name$_size();\n"
-              "}\n",
-              IsImplicitWeakField(field, options_, scc_analyzer_) &&
-                      field->message_type()
-                  ? ".weak"
-                  : "");
-        }
-      } else if (field->real_containing_oneof()) {
-        format.Set("field_name", UnderscoresToCamelCase(field->name(), true));
-        format.Set("oneof_name", field->containing_oneof()->name());
-        format.Set("oneof_index",
-                   StrCat(field->containing_oneof()->index()));
-        GenerateOneofMemberHasBits(field, format);
+
+    // Generate has_$name$() or $name$_size().
+    if (field->is_repeated()) {
+      if (IsFieldStripped(field, options_)) {
+        format(
+            "inline int $classname$::$name$_size() const { "
+            "__builtin_trap(); }\n");
       } else {
-        // Singular field.
-        GenerateSingularFieldHasBits(field, format);
+        format(
+            "inline int $classname$::_internal_$name$_size() const {\n"
+            "  return $field$$1$.size();\n"
+            "}\n"
+            "inline int $classname$::$name$_size() const {\n"
+            "$annotate_size$"
+            "  return _internal_$name$_size();\n"
+            "}\n",
+            IsImplicitWeakField(field, options_, scc_analyzer_) &&
+                    field->message_type()
+                ? ".weak"
+                : "");
       }
+    } else if (field->real_containing_oneof()) {
+      format.Set("field_name", UnderscoresToCamelCase(field->name(), true));
+      format.Set("oneof_name", field->containing_oneof()->name());
+      format.Set("oneof_index",
+                 StrCat(field->containing_oneof()->index()));
+      GenerateOneofMemberHasBits(field, format);
+    } else {
+      // Singular field.
+      GenerateSingularFieldHasBits(field, format);
+    }
 
       if (!IsCrossFileMaybeMap(field)) {
         GenerateFieldClear(field, true, format);
       }
-
     // Generate type-specific accessors.
     if (!IsFieldStripped(field, options_)) {
       field_generators_.get(field).GenerateInlineAccessorDefinitions(printer);
@@ -1760,7 +1759,7 @@ void MessageGenerator::GenerateClassDefinition(io::Printer* printer) {
     format(
         "private:\n"
         "inline bool IsSplitMessageDefault() const {\n"
-        "  return $split$ == reinterpret_cast<Impl_::Split*>(&$1$);\n"
+        "  return $split$ == reinterpret_cast<const Impl_::Split*>(&$1$);\n"
         "}\n"
         "PROTOBUF_NOINLINE void PrepareSplitMessageForWrite();\n"
         "public:\n",
@@ -2423,8 +2422,15 @@ void MessageGenerator::GenerateSharedConstructorCode(io::Printer* printer) {
   }
   if (ShouldSplit(descriptor_, options_)) {
     put_sep();
-    format("decltype($split$){reinterpret_cast<Impl_::Split*>(&$1$)}",
-           DefaultInstanceName(descriptor_, options_, /*split=*/true));
+    // We can't assign the default split to this->split without the const_cast
+    // because the former is a const. The const_cast is safe because we don't
+    // intend to modify the default split through this pointer, and we also
+    // expect the default split to be in the rodata section which is protected
+    // from mutation.
+    format(
+        "decltype($split$){const_cast<Impl_::Split*>"
+        "(reinterpret_cast<const Impl_::Split*>(&$1$))}",
+        DefaultInstanceName(descriptor_, options_, /*split=*/true));
   }
   for (auto oneof : OneOfRange(descriptor_)) {
     put_sep();
@@ -2683,7 +2689,7 @@ void MessageGenerator::GenerateConstexprConstructor(io::Printer* printer) {
   }
   if (ShouldSplit(descriptor_, options_)) {
     put_sep();
-    format("/*decltype($split$)*/&$1$._instance",
+    format("/*decltype($split$)*/const_cast<Impl_::Split*>(&$1$._instance)",
            DefaultInstanceName(descriptor_, options_, /*split=*/true));
   }
 
@@ -2868,8 +2874,10 @@ void MessageGenerator::GenerateStructors(io::Printer* printer) {
       }
       if (ShouldSplit(descriptor_, options_)) {
         put_sep();
-        format("decltype($split$){reinterpret_cast<Impl_::Split*>(&$1$)}",
-               DefaultInstanceName(descriptor_, options_, /*split=*/true));
+        format(
+            "decltype($split$){const_cast<Impl_::Split*>"
+            "(reinterpret_cast<const Impl_::Split*>(&$1$))}",
+            DefaultInstanceName(descriptor_, options_, /*split=*/true));
       }
       for (auto oneof : OneOfRange(descriptor_)) {
         put_sep();

src/google/protobuf/compiler/java/message_serialization.h

@@ -32,6 +32,7 @@
 #define GOOGLE_PROTOBUF_COMPILER_JAVA_MESSAGE_SERIALIZATION_H__
 
 #include <algorithm>
+#include <cstddef>
 #include <vector>
 
 #include <google/protobuf/io/printer.h>
@@ -66,20 +67,31 @@ void GenerateSerializeFieldsAndExtensions(
   std::sort(sorted_extensions.begin(), sorted_extensions.end(),
             ExtensionRangeOrdering());
 
+  std::size_t range_idx = 0;
+
   // Merge the fields and the extension ranges, both sorted by field number.
-  for (int i = 0, j = 0;
-       i < descriptor->field_count() || j < sorted_extensions.size();) {
-    if (i == descriptor->field_count()) {
-      GenerateSerializeExtensionRange(printer, sorted_extensions[j++]);
-    } else if (j == sorted_extensions.size()) {
-      field_generators.get(sorted_fields[i++])
-          .GenerateSerializationCode(printer);
-    } else if (sorted_fields[i]->number() < sorted_extensions[j]->start) {
-      field_generators.get(sorted_fields[i++])
-          .GenerateSerializationCode(printer);
-    } else {
-      GenerateSerializeExtensionRange(printer, sorted_extensions[j++]);
+  for (int i = 0; i < descriptor->field_count(); ++i) {
+    const FieldDescriptor* field = sorted_fields[i];
+
+    // Collapse all extension ranges up until the next field. This leads to
+    // shorter and more efficient codegen for messages containing a large
+    // number of extension ranges without fields in between them.
+    const Descriptor::ExtensionRange* range = nullptr;
+    while (range_idx < sorted_extensions.size() &&
+           sorted_extensions[range_idx]->end <= field->number()) {
+      range = sorted_extensions[range_idx++];
+    }
+
+    if (range != nullptr) {
+      GenerateSerializeExtensionRange(printer, range);
     }
+    field_generators.get(field).GenerateSerializationCode(printer);
+  }
+
+  // After serializing all fields, serialize any remaining extensions via a
+  // single writeUntil call.
+  if (range_idx < sorted_extensions.size()) {
+    GenerateSerializeExtensionRange(printer, sorted_extensions.back());
   }
 }
 

src/google/protobuf/compiler/java/message_serialization_unittest.cc

@@ -0,0 +1,124 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <cstddef>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <google/protobuf/stubs/logging.h>
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/testing/file.h>
+#include <google/protobuf/testing/file.h>
+#include <gmock/gmock.h>
+#include <google/protobuf/testing/googletest.h>
+#include <gtest/gtest.h>
+#include <google/protobuf/compiler/command_line_interface.h>
+#include <google/protobuf/compiler/java/generator.h>
+#include <google/protobuf/test_util2.h>
+
+namespace google {
+namespace protobuf {
+namespace compiler {
+namespace java {
+namespace {
+
+using ::testing::ElementsAre;
+
+// Generates Java code for the specified Java proto, returning the compiler's
+// exit status.
+int CompileJavaProto(std::string proto_file_name) {
+  JavaGenerator java_generator;
+
+  CommandLineInterface cli;
+  cli.RegisterGenerator("--java_out", &java_generator, /*help_text=*/"");
+
+  std::string proto_path = StrCat(
+      "--proto_path=",
+      TestUtil::GetTestDataPath("third_party/protobuf/compiler/java"));
+  std::string java_out = StrCat("--java_out=", TestTempDir());
+
+  const char* argv[] = {
+      "protoc",
+      proto_path.c_str(),
+      java_out.c_str(),
+      proto_file_name.c_str(),
+  };
+
+  // Open-source codebase does not support ABSL_ARRAYSIZE.
+  return cli.Run(sizeof(argv) / sizeof(*argv), argv);
+}
+
+TEST(MessageSerializationTest, CollapseAdjacentExtensionRanges) {
+  GOOGLE_CHECK_EQ(CompileJavaProto("message_serialization_unittest.proto"), 0);
+
+  std::string java_source;
+  GOOGLE_CHECK_OK(File::GetContents(
+      // Open-source codebase does not support file::JoinPath, so we manually
+      // concatenate instead.
+      StrCat(TestTempDir(),
+                   "/TestMessageWithManyExtensionRanges.java"),
+      &java_source, true));
+
+  // Open-source codebase does not support constexpr StringPiece.
+  static constexpr const char kWriteUntilCall[] = "extensionWriter.writeUntil(";
+
+  std::vector<std::string> range_ends;
+
+  // Open-source codebase does not have Split overload taking a single
+  // char delimiter.
+  //
+  // NOLINTNEXTLINE(abseil-faster-strsplit-delimiter)
+  for (const auto& line : Split(java_source, "\n")) {
+    // Extract end position from writeUntil call. (Open-source codebase does not
+    // support RE2.)
+    std::size_t write_until_pos = line.find(kWriteUntilCall);
+    if (write_until_pos == std::string::npos) {
+      continue;
+    }
+    write_until_pos += (sizeof(kWriteUntilCall) - 1);
+
+    std::size_t comma_pos = line.find(',', write_until_pos);
+    if (comma_pos == std::string::npos) {
+      continue;
+    }
+
+    range_ends.push_back(
+        std::string(line.substr(write_until_pos, comma_pos - write_until_pos)));
+  }
+
+  EXPECT_THAT(range_ends, ElementsAre("3", "13", "43"));
+}
+
+}  // namespace
+}  // namespace java
+}  // namespace compiler
+}  // namespace protobuf
+}  // namespace google

src/google/protobuf/compiler/java/message_serialization_unittest.proto

@@ -0,0 +1,56 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto2";
+
+package protobuf_unittest;
+
+option java_multiple_files = true;
+option java_package = "";
+
+// Each batch of extension ranges not separated by a non-extension field should
+// be serialized using a single ExtensionWriter#writeUntil call.
+message TestMessageWithManyExtensionRanges {
+  // First extension range: ends at field number 3 (exclusive)
+  extensions 1 to 2;
+
+  optional int32 foo = 3;
+  optional int32 bar = 5;
+
+  // Second extension range: ends at field number 13 (exclusive)
+  extensions 6;
+  extensions 8;
+  extensions 10 to 12;
+
+  optional int32 baz = 23;
+
+  // Third extension range: ends at field number 43 (exclusive)
+  extensions 42;
+}

src/google/protobuf/compiler/python/pyi_generator.cc

@@ -44,25 +44,10 @@ namespace protobuf {
 namespace compiler {
 namespace python {
 
-template <typename DescriptorT>
-struct SortByName {
-  bool operator()(const DescriptorT* l, const DescriptorT* r) const {
-    return l->name() < r->name();
-  }
-};
-
 PyiGenerator::PyiGenerator() : file_(nullptr) {}
 
 PyiGenerator::~PyiGenerator() {}
 
-void PyiGenerator::PrintItemMap(
-    const std::map<std::string, std::string>& item_map) const {
-  for (const auto& entry : item_map) {
-    printer_->Print("$key$: $value$\n", "key", entry.first, "value",
-                    entry.second);
-  }
-}
-
 template <typename DescriptorT>
 std::string PyiGenerator::ModuleLevelName(const DescriptorT& descriptor) const {
   std::string name = NamePrefixedWithNestedTypes(descriptor, ".");
@@ -177,8 +162,7 @@ void PyiGenerator::PrintImportForDescriptor(
   seen_aliases->insert(alias);
 }
 
-void PyiGenerator::PrintImports(
-    std::map<std::string, std::string>* item_map) const {
+void PyiGenerator::PrintImports() const {
   // Prints imported dependent _pb2 files.
   std::set<std::string> seen_aliases;
   for (int i = 0; i < file_->dependency_count(); ++i) {
@@ -250,7 +234,7 @@ void PyiGenerator::PrintImports(
   if (import_modules.has_union) {
     printer_->Print(", Union as _Union");
   }
-  printer_->Print("\n\n");
+  printer_->Print("\n");
 
   // Public imports
   for (int i = 0; i < file_->public_dependency_count(); ++i) {
@@ -268,17 +252,8 @@ void PyiGenerator::PrintImports(
                       module_name, "enum_class",
                       public_dep->enum_type(i)->name());
     }
-    // Enum values for public imports
-    for (int i = 0; i < public_dep->enum_type_count(); ++i) {
-      const EnumDescriptor* enum_descriptor = public_dep->enum_type(i);
-      for (int j = 0; j < enum_descriptor->value_count(); ++j) {
-        (*item_map)[enum_descriptor->value(j)->name()] =
-            ModuleLevelName(*enum_descriptor);
-      }
-    }
-    // Top level extensions for public imports
-    AddExtensions(*public_dep, item_map);
   }
+printer_->Print("\n");
 }
 
 void PyiGenerator::PrintEnum(const EnumDescriptor& enum_descriptor) const {
@@ -289,19 +264,18 @@ void PyiGenerator::PrintEnum(const EnumDescriptor& enum_descriptor) const {
       "enum_name", enum_name);
 }
 
-// Adds enum value to item map which will be ordered and printed later.
-void PyiGenerator::AddEnumValue(
-    const EnumDescriptor& enum_descriptor,
-    std::map<std::string, std::string>* item_map) const {
+void PyiGenerator::PrintEnumValues(
+    const EnumDescriptor& enum_descriptor) const {
   // enum values
   std::string module_enum_name = ModuleLevelName(enum_descriptor);
   for (int j = 0; j < enum_descriptor.value_count(); ++j) {
     const EnumValueDescriptor* value_descriptor = enum_descriptor.value(j);
-    (*item_map)[value_descriptor->name()] = module_enum_name;
+    printer_->Print("$name$: $module_enum_name$\n",
+                    "name", value_descriptor->name(),
+                    "module_enum_name", module_enum_name);
   }
 }
 
-// Prints top level enums
 void PyiGenerator::PrintTopLevelEnums() const {
   for (int i = 0; i < file_->enum_type_count(); ++i) {
     printer_->Print("\n");
@@ -309,18 +283,16 @@ void PyiGenerator::PrintTopLevelEnums() const {
   }
 }
 
-// Add top level extensions to item_map which will be ordered and
-// printed later.
 template <typename DescriptorT>
-void PyiGenerator::AddExtensions(
-    const DescriptorT& descriptor,
-    std::map<std::string, std::string>* item_map) const {
+void PyiGenerator::PrintExtensions(const DescriptorT& descriptor) const {
   for (int i = 0; i < descriptor.extension_count(); ++i) {
     const FieldDescriptor* extension_field = descriptor.extension(i);
     std::string constant_name = extension_field->name() + "_FIELD_NUMBER";
     ToUpper(&constant_name);
-    (*item_map)[constant_name] = "_ClassVar[int]";
-    (*item_map)[extension_field->name()] = "_descriptor.FieldDescriptor";
+    printer_->Print("$constant_name$: _ClassVar[int]\n",
+                    "constant_name", constant_name);
+    printer_->Print("$name$: _descriptor.FieldDescriptor\n",
+                    "name", extension_field->name());
   }
 }
 
@@ -383,17 +355,11 @@ void PyiGenerator::PrintMessage(
   printer_->Indent();
   printer_->Indent();
 
-  std::vector<const FieldDescriptor*> fields;
-  fields.reserve(message_descriptor.field_count());
-  for (int i = 0; i < message_descriptor.field_count(); ++i) {
-    fields.push_back(message_descriptor.field(i));
-  }
-  std::sort(fields.begin(), fields.end(), SortByName<FieldDescriptor>());
-
   // Prints slots
   printer_->Print("__slots__ = [", "class_name", class_name);
   bool first_item = true;
-  for (const auto& field_des : fields) {
+  for (int i = 0; i < message_descriptor.field_count(); ++i) {
+    const FieldDescriptor* field_des = message_descriptor.field(i);
     if (IsPythonKeyword(field_des->name())) {
       continue;
     }
@@ -406,48 +372,34 @@ void PyiGenerator::PrintMessage(
   }
   printer_->Print("]\n");
 
-  std::map<std::string, std::string> item_map;
   // Prints Extensions for extendable messages
   if (message_descriptor.extension_range_count() > 0) {
-    item_map["Extensions"] = "_python_message._ExtensionDict";
+    printer_->Print("Extensions: _python_message._ExtensionDict\n");
   }
 
   // Prints nested enums
-  std::vector<const EnumDescriptor*> nested_enums;
-  nested_enums.reserve(message_descriptor.enum_type_count());
   for (int i = 0; i < message_descriptor.enum_type_count(); ++i) {
-    nested_enums.push_back(message_descriptor.enum_type(i));
-  }
-  std::sort(nested_enums.begin(), nested_enums.end(),
-            SortByName<EnumDescriptor>());
-
-  for (const auto& entry : nested_enums) {
-    PrintEnum(*entry);
-    // Adds enum value to item_map which will be ordered and printed later
-    AddEnumValue(*entry, &item_map);
+    PrintEnum(*message_descriptor.enum_type(i));
+    PrintEnumValues(*message_descriptor.enum_type(i));
   }
 
   // Prints nested messages
-  std::vector<const Descriptor*> nested_messages;
-  nested_messages.reserve(message_descriptor.nested_type_count());
   for (int i = 0; i < message_descriptor.nested_type_count(); ++i) {
-    nested_messages.push_back(message_descriptor.nested_type(i));
+    PrintMessage(*message_descriptor.nested_type(i), true);
   }
-  std::sort(nested_messages.begin(), nested_messages.end(),
-            SortByName<Descriptor>());
 
-  for (const auto& entry : nested_messages) {
-    PrintMessage(*entry, true);
-  }
+  PrintExtensions(message_descriptor);
 
-  // Adds extensions to item_map which will be ordered and printed later
-  AddExtensions(message_descriptor, &item_map);
-
-  // Adds field number and field descriptor to item_map
+  // Prints field number
+  for (int i = 0; i < message_descriptor.field_count(); ++i) {
+    const FieldDescriptor& field_des = *message_descriptor.field(i);
+    printer_->Print(
+      "$field_number_name$: _ClassVar[int]\n", "field_number_name",
+      ToUpper(field_des.name()) + "_FIELD_NUMBER");
+  }
+  // Prints field name and type
   for (int i = 0; i < message_descriptor.field_count(); ++i) {
     const FieldDescriptor& field_des = *message_descriptor.field(i);
-    item_map[ToUpper(field_des.name()) + "_FIELD_NUMBER"] =
-        "_ClassVar[int]";
     if (IsPythonKeyword(field_des.name())) {
       continue;
     }
@@ -473,12 +425,10 @@ void PyiGenerator::PrintMessage(
     if (field_des.is_repeated()) {
       field_type += "]";
     }
-    item_map[field_des.name()] = field_type;
+    printer_->Print("$name$: $type$\n",
+                    "name", field_des.name(), "type", field_type);
   }
 
-  // Prints all items in item_map
-  PrintItemMap(item_map);
-
   // Prints __init__
   printer_->Print("def __init__(self");
   bool has_key_words = false;
@@ -548,33 +498,19 @@ void PyiGenerator::PrintMessage(
 
 void PyiGenerator::PrintMessages() const {
   // Deterministically order the descriptors.
-  std::vector<const Descriptor*> messages;
-  messages.reserve(file_->message_type_count());
   for (int i = 0; i < file_->message_type_count(); ++i) {
-    messages.push_back(file_->message_type(i));
-  }
-  std::sort(messages.begin(), messages.end(), SortByName<Descriptor>());
-
-  for (const auto& entry : messages) {
-    PrintMessage(*entry, false);
+    PrintMessage(*file_->message_type(i), false);
   }
 }
 
 void PyiGenerator::PrintServices() const {
-  std::vector<const ServiceDescriptor*> services;
-  services.reserve(file_->service_count());
-  for (int i = 0; i < file_->service_count(); ++i) {
-    services.push_back(file_->service(i));
-  }
-  std::sort(services.begin(), services.end(), SortByName<ServiceDescriptor>());
-
   // Prints $Service$ and $Service$_Stub classes
-  for (const auto& entry : services) {
+  for (int i = 0; i < file_->service_count(); ++i) {
     printer_->Print("\n");
     printer_->Print(
         "class $service_name$(_service.service): ...\n\n"
         "class $service_name$_Stub($service_name$): ...\n",
-        "service_name", entry->name());
+        "service_name", file_->service(i)->name());
   }
 }
 
@@ -594,25 +530,28 @@ bool PyiGenerator::Generate(const FileDescriptor* file,
   io::Printer printer(output.get(), '$');
   printer_ = &printer;
 
-  // item map will store "DESCRIPTOR", top level extensions, top level enum
-  // values. The items will be sorted and printed later.
-  std::map<std::string, std::string> item_map;
+  PrintImports();
+  printer_->Print("DESCRIPTOR: _descriptor.FileDescriptor\n");
 
-  // Adds "DESCRIPTOR" into item_map.
-  item_map["DESCRIPTOR"] = "_descriptor.FileDescriptor";
+  // Prints extensions and enums from imports.
+  for (int i = 0; i < file_->public_dependency_count(); ++i) {
+    const FileDescriptor* public_dep = file_->public_dependency(i);
+    PrintExtensions(*public_dep);
+    for (int i = 0; i < public_dep->enum_type_count(); ++i) {
+      const EnumDescriptor* enum_descriptor = public_dep->enum_type(i);
+      PrintEnumValues(*enum_descriptor);
+    }
+  }
 
-  PrintImports(&item_map);
-  // Adds top level enum values to item_map.
+  PrintTopLevelEnums();
+  // Prints top level enum values
   for (int i = 0; i < file_->enum_type_count(); ++i) {
-    AddEnumValue(*file_->enum_type(i), &item_map);
+    PrintEnumValues(*file_->enum_type(i));
   }
-  // Adds top level extensions to item_map.
-  AddExtensions(*file_, &item_map);
-  // Prints item map
-  PrintItemMap(item_map);
-
+  // Prints top level Extensions
+  PrintExtensions(*file_);
   PrintMessages();
-  PrintTopLevelEnums();
+
   if (HasGenericServices(file)) {
     PrintServices();
   }

src/google/protobuf/compiler/python/pyi_generator.h

@@ -77,18 +77,15 @@ class PROTOC_EXPORT PyiGenerator : public google::protobuf::compiler::CodeGenera
  private:
   void PrintImportForDescriptor(const FileDescriptor& desc,
                                 std::set<std::string>* seen_aliases) const;
-  void PrintImports(std::map<std::string, std::string>* item_map) const;
-  void PrintEnum(const EnumDescriptor& enum_descriptor) const;
-  void AddEnumValue(const EnumDescriptor& enum_descriptor,
-                    std::map<std::string, std::string>* item_map) const;
+  void PrintImports() const;
   void PrintTopLevelEnums() const;
+  void PrintEnum(const EnumDescriptor& enum_descriptor) const;
+  void PrintEnumValues(const EnumDescriptor& enum_descriptor) const;
   template <typename DescriptorT>
-  void AddExtensions(const DescriptorT& descriptor,
-                     std::map<std::string, std::string>* item_map) const;
+  void PrintExtensions(const DescriptorT& descriptor) const;
   void PrintMessages() const;
   void PrintMessage(const Descriptor& message_descriptor, bool is_nested) const;
   void PrintServices() const;
-  void PrintItemMap(const std::map<std::string, std::string>& item_map) const;
   std::string GetFieldType(
       const FieldDescriptor& field_des, const Descriptor& containing_des) const;
   template <typename DescriptorT>

src/google/protobuf/generated_message_tctable_impl.h

@@ -38,8 +38,6 @@
 #include <google/protobuf/port.h>
 #include <google/protobuf/extension_set.h>
 #include <google/protobuf/generated_message_tctable_decl.h>
-#include <google/protobuf/generated_message_tctable_impl.h>
-#include <google/protobuf/message_lite.h>
 #include <google/protobuf/metadata_lite.h>
 #include <google/protobuf/parse_context.h>
 #include <google/protobuf/wire_format_lite.h>

src/google/protobuf/io/printer.h

@@ -251,7 +251,8 @@ class PROTOBUF_EXPORT Printer {
   template <typename... Args>
   void Print(const char* text, const Args&... args) {
     std::map<std::string, std::string> vars;
-    PrintInternal(&vars, text, args...);
+    FillMap(&vars, args...);
+    Print(vars, text);
   }
 
   // Indent text by two spaces.  After calling Indent(), two spaces will be
@@ -299,18 +300,13 @@ class PROTOBUF_EXPORT Printer {
   void Annotate(const char* begin_varname, const char* end_varname,
                 const std::string& file_path, const std::vector<int>& path);
 
-  // Base case
-  void PrintInternal(std::map<std::string, std::string>* vars,
-                     const char* text) {
-    Print(*vars, text);
-  }
+  void FillMap(std::map<std::string, std::string>* vars) {}
 
   template <typename... Args>
-  void PrintInternal(std::map<std::string, std::string>* vars, const char* text,
-                     const char* key, const std::string& value,
-                     const Args&... args) {
+  void FillMap(std::map<std::string, std::string>* vars, const std::string& key,
+               const std::string& value, const Args&... args) {
     (*vars)[key] = value;
-    PrintInternal(vars, text, args...);
+    FillMap(vars, args...);
   }
 
   // Copy size worth of bytes from data to buffer_.

src/google/protobuf/map.h

@@ -333,6 +333,11 @@ inline size_t SpaceUsedInValues(const void*) { return 0; }
 
 }  // namespace internal
 
+#ifdef PROTOBUF_FUTURE_MAP_PAIR_UPGRADE
+// This is the class for Map's internal value_type.
+template <typename Key, typename T>
+using MapPair = std::pair<const Key, T>;
+#else
 // This is the class for Map's internal value_type. Instead of using
 // std::pair as value_type, we use this class which provides us more control of
 // its process of construction and destruction.
@@ -363,6 +368,7 @@ struct PROTOBUF_ATTRIBUTE_STANDALONE_DEBUG MapPair {
   friend class Arena;
   friend class Map<Key, T>;
 };
+#endif
 
 // Map is an associative container type used to store protobuf map
 // fields.  Each Map instance may or may not use a different hash function, a

src/google/protobuf/map_test.cc

@@ -84,3 +84,5 @@ TEST(MapTest, Aligned8OnArena) { MapTest_Aligned<AlignedAs8, true>(); }
 }  // namespace internal
 }  // namespace protobuf
 }  // namespace google
+
+#include <google/protobuf/port_undef.inc>

src/google/protobuf/map_test.inc

@@ -810,6 +810,8 @@ TEST_F(MapImplTest, Emplace) {
       m, UnorderedElementsAre(Pair(1, "one"), Pair(2, "two"), Pair(42, "aaa")));
 }
 
+#ifndef PROTOBUF_FUTURE_MAP_PAIR_UPGRADE
+
 TEST_F(MapImplTest, EmplaceKeyOnly) {
   using ::testing::Pair;
   using ::testing::UnorderedElementsAre;
@@ -824,6 +826,43 @@ TEST_F(MapImplTest, EmplaceKeyOnly) {
   EXPECT_THAT(m, UnorderedElementsAre(Pair(1, ""), Pair(42, "")));
 }
 
+#else
+
+TEST_F(MapImplTest, ValueTypeNoImplicitConversion) {
+  using vt = typename Map<const char*, int>::value_type;
+
+  EXPECT_FALSE((std::is_convertible<
+                vt, std::pair<std::string, std::vector<std::string>>>::value));
+}
+
+enum class ConstructorType {
+  kDefault,
+  kCopy,
+  kMove,
+};
+
+struct ConstructorTag {
+  ConstructorTag() : invoked_constructor(ConstructorType::kDefault) {}
+  ConstructorTag(const ConstructorTag&)
+      : invoked_constructor(ConstructorType::kCopy) {}
+  ConstructorTag(ConstructorTag&&)
+      : invoked_constructor(ConstructorType::kMove) {}
+
+  ConstructorType invoked_constructor;
+};
+
+TEST_F(MapImplTest, ValueTypeHasMoveConstructor) {
+  using vt = typename Map<ConstructorTag, ConstructorTag>::value_type;
+  ConstructorTag l, r;
+
+  vt pair(l, std::move(r));
+
+  EXPECT_EQ(pair.first.invoked_constructor, ConstructorType::kCopy);
+  EXPECT_EQ(pair.second.invoked_constructor, ConstructorType::kMove);
+}
+
+#endif  // !PROTOBUF_FUTURE_MAP_PAIR_UPGRADE
+
 struct CountedInstance {
   CountedInstance() { ++num_created; }
   CountedInstance(const CountedInstance&) : CountedInstance() {}

src/google/protobuf/message.cc

@@ -214,7 +214,7 @@ uint64_t Message::GetInvariantPerBuild(uint64_t salt) {
 }
 
 namespace internal {
-void* CreateSplitMessageGeneric(Arena* arena, void* default_split,
+void* CreateSplitMessageGeneric(Arena* arena, const void* default_split,
                                 size_t size) {
   void* split =
       (arena == nullptr) ? ::operator new(size) : arena->AllocateAligned(size);

src/google/protobuf/message.h

@@ -411,7 +411,8 @@ class PROTOBUF_EXPORT Message : public MessageLite {
 
 namespace internal {
 // Creates and returns an allocation for a split message.
-void* CreateSplitMessageGeneric(Arena* arena, void* default_split, size_t size);
+void* CreateSplitMessageGeneric(Arena* arena, const void* default_split,
+                                size_t size);
 
 // Forward-declare interfaces used to implement RepeatedFieldRef.
 // These are protobuf internals that users shouldn't care about.

src/google/protobuf/port_def.inc

@@ -181,11 +181,15 @@
 // Future versions of protobuf will include breaking changes to some APIs.
 // This macro can be set to enable these API changes ahead of time, so that
 // user code can be updated before upgrading versions of protobuf.
-// PROTOBUF_FUTURE_FINAL is used on classes that are historically not marked as
-// final, but that may be marked final in future (breaking) releases.
 
 #ifdef PROTOBUF_FUTURE_BREAKING_CHANGES
-// Used on classes that are historically not marked as final.
+
+// Used to upgrade google::protobuf::MapPair<K, V> to std::pair<const K, V>.
+// Owner: mordberg@
+#define PROTOBUF_FUTURE_MAP_PAIR_UPGRADE 1
+
+// Used on classes that are historically not marked as final, but that may be
+// marked final in future (breaking) releases.
 // Owner: kfm@
 #define PROTOBUF_FUTURE_FINAL final
 

src/google/protobuf/port_undef.inc

@@ -113,6 +113,7 @@
 
 #ifdef PROTOBUF_FUTURE_BREAKING_CHANGES
 #undef PROTOBUF_FUTURE_BREAKING_CHANGES
+#undef PROTOBUF_FUTURE_MAP_PAIR_UPGRADE
 #undef PROTOBUF_FUTURE_REMOVE_DEFAULT_FIELD_COMPARATOR
 #undef PROTOBUF_FUTURE_REMOVE_CLEARED_API
 #endif

src/google/protobuf/repeated_field.h

@@ -233,12 +233,6 @@ class RepeatedField final {
   // copies data between each other.
   void Swap(RepeatedField* other);
 
-  // Swaps entire contents with "other". Should be called only if the caller can
-  // guarantee that both repeated fields are on the same arena or are on the
-  // heap. Swapping between different arenas is disallowed and caught by a
-  // GOOGLE_DCHECK (see API docs for details).
-  void UnsafeArenaSwap(RepeatedField* other);
-
   // Swaps two elements.
   void SwapElements(int index1, int index2);
 
@@ -321,6 +315,12 @@ class RepeatedField final {
                               : rep()->arena;
   }
 
+  // Swaps entire contents with "other". Should be called only if the caller can
+  // guarantee that both repeated fields are on the same arena or are on the
+  // heap. Swapping between different arenas is disallowed and caught by a
+  // GOOGLE_DCHECK (see API docs for details).
+  void UnsafeArenaSwap(RepeatedField* other);
+
   static constexpr int kInitialSize = 0;
   // A note on the representation here (see also comment below for
   // RepeatedPtrFieldBase's struct Rep):

src/google/protobuf/util/json_util_test.cc

@@ -307,6 +307,15 @@ TEST_P(JsonTest, Camels) {
   EXPECT_THAT(ToJson(m), IsOkAndHolds(R"({"StringField":"sTRINGfIELD"})"));
 }
 
+TEST_P(JsonTest, EvilString) {
+  auto m = ToProto<TestMessage>(R"json(
+    {"string_value": ")json"
+                                "\n\r\b\f\1\2\3"
+                                "\"}");
+  ASSERT_OK(m);
+  EXPECT_EQ(m->string_value(), "\n\r\b\f\1\2\3");
+}
+
 TEST_P(JsonTest, TestAlwaysPrintEnumsAsInts) {
   TestMessage orig;
   orig.set_enum_value(proto3::BAR);
@@ -399,6 +408,7 @@ TEST_P(JsonTest, ParseMessage) {
       "repeatedEnumValue": [1, "FOO"],
       "repeatedMessageValue": [
         {"value": 40},
+        {},
         {"value": 96}
       ]
     }
@@ -427,9 +437,10 @@ TEST_P(JsonTest, ParseMessage) {
   EXPECT_THAT(m->repeated_string_value(), ElementsAre("foo", "bar ", ""));
   EXPECT_THAT(m->repeated_enum_value(), ElementsAre(proto3::BAR, proto3::FOO));
 
-  ASSERT_THAT(m->repeated_message_value(), SizeIs(2));
+  ASSERT_THAT(m->repeated_message_value(), SizeIs(3));
   EXPECT_EQ(m->repeated_message_value(0).value(), 40);
-  EXPECT_EQ(m->repeated_message_value(1).value(), 96);
+  EXPECT_EQ(m->repeated_message_value(1).value(), 0);
+  EXPECT_EQ(m->repeated_message_value(2).value(), 96);
 
   EXPECT_THAT(
       ToJson(*m),
@@ -440,7 +451,7 @@ TEST_P(JsonTest, ParseMessage) {
           R"("messageValue":{"value":2048},"repeatedBoolValue":[true],"repeatedInt32Value":[0,-42])"
           R"(,"repeatedUint64Value":["1","2"],"repeatedDoubleValue":[1.5,-2],)"
           R"("repeatedStringValue":["foo","bar ",""],"repeatedEnumValue":["BAR","FOO"],)"
-          R"("repeatedMessageValue":[{"value":40},{"value":96}]})"));
+          R"("repeatedMessageValue":[{"value":40},{},{"value":96}]})"));
 }
 
 TEST_P(JsonTest, CurseOfAtob) {
@@ -761,6 +772,24 @@ TEST_P(JsonTest, TestFlatList) {
   )json");
   ASSERT_OK(m);
   EXPECT_THAT(m->repeated_int32_value(), ElementsAre(5, 6));
+
+  // The above flatteing behavior is supressed for google::protobuf::ListValue.
+  auto m2 = ToProto<google::protobuf::Value>(R"json(
+    {
+      "repeatedInt32Value": [[[5]], [6]]
+    }
+  )json");
+  ASSERT_OK(m2);
+  auto fields = m2->struct_value().fields();
+  auto list = fields["repeatedInt32Value"].list_value();
+  EXPECT_EQ(list.values(0)
+                .list_value()
+                .values(0)
+                .list_value()
+                .values(0)
+                .number_value(),
+            5);
+  EXPECT_EQ(list.values(1).list_value().values(0).number_value(), 6);
 }
 
 TEST_P(JsonTest, ParseWrappers) {
@@ -1095,6 +1124,33 @@ TEST_P(JsonTest, TestLegalNullsInArray) {
 
   ASSERT_THAT(m2->repeated_value(), SizeIs(1));
   EXPECT_TRUE(m2->repeated_value(0).has_null_value());
+
+  m2->Clear();
+  m2->mutable_repeated_value();  // Materialize an empty singular Value.
+  m2->add_repeated_value();
+  m2->add_repeated_value()->set_string_value("solitude");
+  m2->add_repeated_value();
+  EXPECT_THAT(ToJson(*m2), IsOkAndHolds(R"({"repeatedValue":["solitude"]})"));
+}
+
+TEST_P(JsonTest, ListList) {
+  auto m = ToProto<proto3::TestListValue>(R"json({
+    "repeated_value": [["ayy", "lmao"]]
+  })json");
+  ASSERT_OK(m);
+
+  EXPECT_EQ(m->repeated_value(0).values(0).string_value(), "ayy");
+  EXPECT_EQ(m->repeated_value(0).values(1).string_value(), "lmao");
+
+  m = ToProto<proto3::TestListValue>(R"json({
+    "repeated_value": [{
+      "values": ["ayy", "lmao"]
+    }]
+  })json");
+  ASSERT_OK(m);
+
+  EXPECT_EQ(m->repeated_value(0).values(0).string_value(), "ayy");
+  EXPECT_EQ(m->repeated_value(0).values(1).string_value(), "lmao");
 }
 
 TEST_P(JsonTest, HtmlEscape) {