Down integrate to Github

cmake/CMakeLists.txt

@@ -80,11 +80,24 @@ string(REGEX REPLACE     "${protobuf_VERSION_REGEX}" "\\5"
 
 message(STATUS "${protobuf_VERSION_PRERELEASE}")
 
+message(STATUS "${protobuf_VERSION_PRERELEASE}")
+
 # Package version
 set(protobuf_VERSION
   "${protobuf_VERSION_MAJOR}.${protobuf_VERSION_MINOR}.${protobuf_VERSION_PATCH}")
 
 if(protobuf_VERSION_PRERELEASE)
+<<<<<<<
+=======
+  set(protobuf_VERSION "${protobuf_VERSION}.${protobuf_VERSION_PRERELEASE}")
+else()
+  set(protobuf_VERSION "${protobuf_VERSION}.0")
+endif()
+message(STATUS "${protobuf_VERSION}")
+
+if(protobuf_VERBOSE)
+
+>>>>>>>
   set(protobuf_VERSION "${protobuf_VERSION}.${protobuf_VERSION_PRERELEASE}")
 else()
   set(protobuf_VERSION "${protobuf_VERSION}.0")

cmake/extract_includes.bat.in

@@ -115,7 +115,6 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\type_resolver.h"
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\type_resolver_util.h" include\google\protobuf\util\type_resolver_util.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wire_format.h" include\google\protobuf\wire_format.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wire_format_lite.h" include\google\protobuf\wire_format_lite.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wire_format_lite_inl.h" include\google\protobuf\wire_format_lite_inl.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wrappers.pb.h" include\google\protobuf\wrappers.pb.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\any.proto" include\google\protobuf\any.proto
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\api.proto" include\google\protobuf\api.proto

conformance/conformance.proto

@@ -119,6 +119,10 @@ message ConformanceRequest {
 
   // Specify details for how to encode jspb.
   JspbEncodingConfig jspb_encoding_options = 6;
+
+  // This can be used in json and text format. If true, testee should print
+  // unknown fields instead of ignore. This feature is optional.
+  bool print_unknown_fields = 9;
 }
 
 // Represents a single test case's output.

conformance/conformance_cpp.cc

@@ -214,8 +214,10 @@ void DoTest(const ConformanceRequest& request, ConformanceResponse* response) {
     }
 
     case conformance::TEXT_FORMAT: {
-      GOOGLE_CHECK(TextFormat::PrintToString(*test_message,
+      TextFormat::Printer printer;
-                                             response->mutable_text_payload()));
+      printer.SetHideUnknownFields(!request.print_unknown_fields());
+      GOOGLE_CHECK(printer.PrintToString(*test_message,
+                                         response->mutable_text_payload()));
       break;
     }
 

conformance/conformance_python.py

@@ -166,7 +166,8 @@ def do_test(request):
         return response
 
     elif request.requested_output_format == conformance_pb2.TEXT_FORMAT:
-      response.text_payload = text_format.MessageToString(test_message)
+      response.text_payload = text_format.MessageToString(
+          test_message, print_unknown_fields=request.print_unknown_fields)
 
   except Exception as e:
     response.runtime_error = str(e)

conformance/conformance_test.cc

@@ -68,6 +68,7 @@ ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting(
       input_format_(input_format),
       output_format_(output_format),
       prototype_message_(prototype_message),
+      prototype_message_for_compare_(prototype_message.New()),
       test_name_(test_name) {
   switch (input_format) {
     case conformance::PROTOBUF: {
@@ -102,7 +103,7 @@ ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting(
 
 Message* ConformanceTestSuite::ConformanceRequestSetting::
     GetTestMessage() const {
-  return prototype_message_.New();
+  return prototype_message_for_compare_->New();
 }
 
 string ConformanceTestSuite::ConformanceRequestSetting::

conformance/conformance_test.h

@@ -224,6 +224,14 @@ class ConformanceTestSuite {
 
     string ConformanceLevelToString(ConformanceLevel level) const;
 
+    void SetPrintUnknownFields(bool print_unknown_fields) {
+      request_.set_print_unknown_fields(true);
+    }
+
+    void SetPrototypeMessageForCompare(const Message& message) {
+      prototype_message_for_compare_.reset(message.New());
+    }
+
    protected:
     virtual string InputFormatString(conformance::WireFormat format) const;
     virtual string OutputFormatString(conformance::WireFormat format) const;
@@ -234,6 +242,7 @@ class ConformanceTestSuite {
     ::conformance::WireFormat input_format_;
     ::conformance::WireFormat output_format_;
     const Message& prototype_message_;
+    std::unique_ptr<Message> prototype_message_for_compare_;
     string test_name_;
   };
 

conformance/text_format_conformance_suite.cc

@@ -43,6 +43,7 @@ using conformance::WireFormat;
 using google::protobuf::Message;
 using google::protobuf::TextFormat;
 using protobuf_test_messages::proto2::TestAllTypesProto2;
+using protobuf_test_messages::proto2::UnknownToTestAllTypes;
 using protobuf_test_messages::proto3::TestAllTypesProto3;
 using std::string;
 
@@ -54,8 +55,14 @@ TextFormatConformanceTestSuite::TextFormatConformanceTestSuite() {
 }
 
 bool TextFormatConformanceTestSuite::ParseTextFormatResponse(
-    const ConformanceResponse& response, Message* test_message) {
+    const ConformanceResponse& response,
-  if (!TextFormat::ParseFromString(response.text_payload(), test_message)) {
+    const ConformanceRequestSetting& setting, Message* test_message) {
+  TextFormat::Parser parser;
+  const ConformanceRequest& request = setting.GetRequest();
+  if (request.print_unknown_fields()) {
+    parser.AllowFieldNumber(true);
+  }
+  if (!parser.ParseFromString(response.text_payload(), test_message)) {
     GOOGLE_LOG(ERROR) << "INTERNAL ERROR: internal text->protobuf transcode "
                       << "yielded unparseable proto. Text payload: "
                       << response.text_payload();
@@ -103,7 +110,7 @@ bool TextFormatConformanceTestSuite::ParseResponse(
         return false;
       }
 
-      if (!ParseTextFormatResponse(response, test_message)) {
+      if (!ParseTextFormatResponse(response, setting, test_message)) {
         ReportFailure(
             test_name, level, request, response,
             "TEXT_FORMAT output we received from test was unparseable.");
@@ -171,6 +178,27 @@ void TextFormatConformanceTestSuite::RunValidTextFormatTestWithMessage(
   RunValidInputTest(setting2, input_text);
 }
 
+void TextFormatConformanceTestSuite::RunValidUnknownTextFormatTest(
+    const string& test_name, const Message& message) {
+  string serialized_input;
+  message.SerializeToString(&serialized_input);
+  TestAllTypesProto3 prototype;
+  ConformanceRequestSetting setting1(
+      RECOMMENDED, conformance::PROTOBUF, conformance::TEXT_FORMAT,
+      conformance::TEXT_FORMAT_TEST, prototype, test_name + "_Drop",
+      serialized_input);
+  setting1.SetPrototypeMessageForCompare(message);
+  RunValidBinaryInputTest(setting1, "");
+
+  ConformanceRequestSetting setting2(
+      RECOMMENDED, conformance::PROTOBUF, conformance::TEXT_FORMAT,
+      conformance::TEXT_FORMAT_TEST, prototype, test_name + "_Print",
+      serialized_input);
+  setting2.SetPrototypeMessageForCompare(message);
+  setting2.SetPrintUnknownFields(true);
+  RunValidBinaryInputTest(setting2, serialized_input);
+}
+
 void TextFormatConformanceTestSuite::RunSuiteImpl() {
   RunValidTextFormatTest("HelloWorld", REQUIRED,
                          "optional_string: 'Hello, World!'");
@@ -235,6 +263,29 @@ void TextFormatConformanceTestSuite::RunSuiteImpl() {
                                "Data: { group_int32: 1 }");
   RunValidTextFormatTestProto2("GroupFieldEmpty", REQUIRED,
                                "Data {}");
+
+
+  // Unknown Fields
+  UnknownToTestAllTypes message;
+  // Unable to print unknown Fixed32/Fixed64 fields as if they are known.
+  // Fixed32/Fixed64 fields are not added in the tests.
+  message.set_optional_int32(123);
+  message.set_optional_string("hello");
+  message.set_optional_bool(true);
+  RunValidUnknownTextFormatTest("ScalarUnknownFields", message);
+
+  message.Clear();
+  message.mutable_nested_message()->set_c(111);
+  RunValidUnknownTextFormatTest("MessageUnknownFields", message);
+
+  message.Clear();
+  message.mutable_optionalgroup()->set_a(321);
+  RunValidUnknownTextFormatTest("GroupUnknownFields", message);
+
+  message.add_repeated_int32(1);
+  message.add_repeated_int32(2);
+  message.add_repeated_int32(3);
+  RunValidUnknownTextFormatTest("RepeatedUnknownFields", message);
 }
 
 }  // namespace protobuf

conformance/text_format_conformance_suite.h

@@ -51,9 +51,12 @@ class TextFormatConformanceTestSuite : public ConformanceTestSuite {
                                          ConformanceLevel level,
                                          const string& input_text,
                                          const Message& prototype);
+  void RunValidUnknownTextFormatTest(const string& test_name,
+                                     const Message& message);
   void ExpectParseFailure(const string& test_name, ConformanceLevel level,
                           const string& input);
   bool ParseTextFormatResponse(const conformance::ConformanceResponse& response,
+                               const ConformanceRequestSetting& setting,
                                Message* test_message);
   bool ParseResponse(const conformance::ConformanceResponse& response,
                      const ConformanceRequestSetting& setting,

conformance/text_format_failure_list_java.txt

@@ -0,0 +1,4 @@
+Recommended.Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput

java/core/src/main/java/com/google/protobuf/ByteString.java

@@ -1438,16 +1438,14 @@ public abstract class ByteString implements Iterable<Byte>, Serializable {
         LiteralByteString lbsOther = (LiteralByteString) other;
         byte[] thisBytes = bytes;
         byte[] otherBytes = lbsOther.bytes;
-        int thisLimit = getOffsetIntoBytes() + length;
+
-        for (int thisIndex = getOffsetIntoBytes(),
+        return UnsafeUtil.mismatch(
-                otherIndex = lbsOther.getOffsetIntoBytes() + offset;
+                thisBytes,
-            (thisIndex < thisLimit);
+                getOffsetIntoBytes(),
-            ++thisIndex, ++otherIndex) {
+                otherBytes,
-          if (thisBytes[thisIndex] != otherBytes[otherIndex]) {
+                lbsOther.getOffsetIntoBytes() + offset,
-            return false;
+                length)
-          }
+            == -1;
-        }
-        return true;
       }
 
       return other.substring(offset, offset + length).equals(substring(0, length));

java/core/src/main/java/com/google/protobuf/CodedOutputStream.java

@@ -1264,16 +1264,34 @@ public abstract class CodedOutputStream extends ByteOutput {
 
     @Override
     public final void writeUInt32NoTag(int value) throws IOException {
-      if (HAS_UNSAFE_ARRAY_OPERATIONS && spaceLeft() >= MAX_VARINT32_SIZE) {
+      if (HAS_UNSAFE_ARRAY_OPERATIONS
-        while (true) {
+          && !Android.isOnAndroidDevice()
-          if ((value & ~0x7F) == 0) {
+          && spaceLeft() >= MAX_VARINT32_SIZE) {
-            UnsafeUtil.putByte(buffer, position++, (byte) value);
+        if ((value & ~0x7F) == 0) {
-            return;
+          UnsafeUtil.putByte(buffer, position++, (byte) value);
-          } else {
+          return;
-            UnsafeUtil.putByte(buffer, position++, (byte) ((value & 0x7F) | 0x80));
+        }
-            value >>>= 7;
+        UnsafeUtil.putByte(buffer, position++, (byte) (value | 0x80));
-          }
+        value >>>= 7;
+        if ((value & ~0x7F) == 0) {
+          UnsafeUtil.putByte(buffer, position++, (byte) value);
+          return;
+        }
+        UnsafeUtil.putByte(buffer, position++, (byte) (value | 0x80));
+        value >>>= 7;
+        if ((value & ~0x7F) == 0) {
+          UnsafeUtil.putByte(buffer, position++, (byte) value);
+          return;
+        }
+        UnsafeUtil.putByte(buffer, position++, (byte) (value | 0x80));
+        value >>>= 7;
+        if ((value & ~0x7F) == 0) {
+          UnsafeUtil.putByte(buffer, position++, (byte) value);
+          return;
         }
+        UnsafeUtil.putByte(buffer, position++, (byte) (value | 0x80));
+        value >>>= 7;
+        UnsafeUtil.putByte(buffer, position++, (byte) value);
       } else {
         try {
           while (true) {

java/core/src/main/java/com/google/protobuf/Extension.java

@@ -30,6 +30,7 @@
 
 package com.google.protobuf;
 
+// TODO(chrisn): Change ContainingType to extend Message
 /**
  * Interface that generated extensions implement.
  *
@@ -37,6 +38,11 @@ package com.google.protobuf;
  */
 public abstract class Extension<ContainingType extends MessageLite, Type>
     extends ExtensionLite<ContainingType, Type> {
+  // TODO(chrisn): Add package-private constructor.
+
+  /** {@inheritDoc} Overridden to return {@link Message} instead of {@link MessageLite}. */
+  @Override
+  public abstract Message getMessageDefaultInstance();
 
   /** Returns the descriptor of the extension. */
   public abstract Descriptors.FieldDescriptor getDescriptor();

java/util/src/main/java/com/google/protobuf/util/JsonFormat.java

@@ -448,22 +448,26 @@ public class JsonFormat {
     }
 
     /**
-     * Find a type by its full name. Returns null if it cannot be found in
+     * Find a type by its full name. Returns null if it cannot be found in this {@link
-     * this {@link TypeRegistry}.
+     * TypeRegistry}.
      */
     public Descriptor find(String name) {
       return types.get(name);
     }
 
+    /* @Nullable */
+    Descriptor getDescriptorForTypeUrl(String typeUrl) throws InvalidProtocolBufferException {
+      return find(getTypeName(typeUrl));
+    }
+
     private final Map<String, Descriptor> types;
 
     private TypeRegistry(Map<String, Descriptor> types) {
       this.types = types;
     }
 
-    /**
+
-     * A Builder is used to build {@link TypeRegistry}.
+    /** A Builder is used to build {@link TypeRegistry}. */
-     */
     public static class Builder {
       private Builder() {}
 
@@ -801,15 +805,14 @@ public class JsonFormat {
         throw new InvalidProtocolBufferException("Invalid Any type.");
       }
       String typeUrl = (String) message.getField(typeUrlField);
-      String typeName = getTypeName(typeUrl);
+      Descriptor type = registry.getDescriptorForTypeUrl(typeUrl);
-      Descriptor type = registry.find(typeName);
       if (type == null) {
         throw new InvalidProtocolBufferException("Cannot find type for url: " + typeUrl);
       }
       ByteString content = (ByteString) message.getField(valueField);
       Message contentMessage =
           DynamicMessage.getDefaultInstance(type).getParserForType().parseFrom(content);
-      WellKnownTypePrinter printer = wellKnownTypePrinters.get(typeName);
+      WellKnownTypePrinter printer = wellKnownTypePrinters.get(getTypeName(typeUrl));
       if (printer != null) {
         // If the type is one of the well-known types, we use a special
         // formatting.
@@ -1443,7 +1446,7 @@ public class JsonFormat {
         throw new InvalidProtocolBufferException("Missing type url when parsing: " + json);
       }
       String typeUrl = typeUrlElement.getAsString();
-      Descriptor contentType = registry.find(getTypeName(typeUrl));
+      Descriptor contentType = registry.getDescriptorForTypeUrl(typeUrl);
       if (contentType == null) {
         throw new InvalidProtocolBufferException("Cannot resolve type: " + typeUrl);
       }

java/util/src/main/java/com/google/protobuf/util/Timestamps.java

@@ -112,8 +112,9 @@ public final class Timestamps {
       };
 
   /**
-   * Returns a {@link Comparator} for {@link Timestamp}s which sorts in increasing chronological
+   * Returns a {@link Comparator} for {@link Timestamp Timestamps} which sorts in increasing
-   * order. Nulls and invalid {@link Timestamp}s are not allowed (see {@link #isValid}).
+   * chronological order. Nulls and invalid {@link Timestamp Timestamps} are not allowed (see
+   * {@link #isValid}).
    */
   public static Comparator<Timestamp> comparator() {
     return COMPARATOR;

java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java

@@ -34,6 +34,7 @@ import com.google.protobuf.Any;
 import com.google.protobuf.BoolValue;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.BytesValue;
+import com.google.protobuf.Descriptors.Descriptor;
 import com.google.protobuf.Descriptors.FieldDescriptor;
 import com.google.protobuf.DoubleValue;
 import com.google.protobuf.FloatValue;
@@ -834,6 +835,7 @@ public class JsonFormatTest extends TestCase {
     assertRoundTripEquals(message);
   }
 
+
   public void testAnyFields() throws Exception {
     TestAllTypes content = TestAllTypes.newBuilder().setOptionalInt32(1234).build();
     TestAny message = TestAny.newBuilder().setAnyValue(Any.pack(content)).build();

js/data.proto

@@ -32,11 +32,11 @@
 
 syntax = "proto2";
 
+package jspb.test;
+
 option java_package = "com.google.apps.jspb.proto";
 option java_multiple_files = true;
 
-package jspb.test;
-
 // legacy data, must be nested
 message data {
   message NestedData {
@@ -48,4 +48,3 @@ message data {
 message UnnestedData {
   required string str = 1;
 }
-

js/proto3_test.proto

@@ -35,43 +35,43 @@ import "testbinary.proto";
 package jspb.test;
 
 message TestProto3 {
-     int32 optional_int32    =  1;
+  int32 optional_int32 = 1;
-     int64 optional_int64    =  2;
+  int64 optional_int64 = 2;
-    uint32 optional_uint32   =  3;
+  uint32 optional_uint32 = 3;
-    uint64 optional_uint64   =  4;
+  uint64 optional_uint64 = 4;
-    sint32 optional_sint32   =  5;
+  sint32 optional_sint32 = 5;
-    sint64 optional_sint64   =  6;
+  sint64 optional_sint64 = 6;
-   fixed32 optional_fixed32  =  7;
+  fixed32 optional_fixed32 = 7;
-   fixed64 optional_fixed64  =  8;
+  fixed64 optional_fixed64 = 8;
-  sfixed32 optional_sfixed32 =  9;
+  sfixed32 optional_sfixed32 = 9;
   sfixed64 optional_sfixed64 = 10;
-     float optional_float    = 11;
+  float optional_float = 11;
-    double optional_double   = 12;
+  double optional_double = 12;
-      bool optional_bool     = 13;
+  bool optional_bool = 13;
-    string optional_string   = 14;
+  string optional_string = 14;
-     bytes optional_bytes    = 15;
+  bytes optional_bytes = 15;
 
   ForeignMessage optional_foreign_message = 19;
-  Proto3Enum     optional_foreign_enum    = 22;
+  Proto3Enum optional_foreign_enum = 22;
 
-  repeated    int32 repeated_int32    = 31;
+  repeated int32 repeated_int32 = 31;
-  repeated    int64 repeated_int64    = 32;
+  repeated int64 repeated_int64 = 32;
-  repeated   uint32 repeated_uint32   = 33;
+  repeated uint32 repeated_uint32 = 33;
-  repeated   uint64 repeated_uint64   = 34;
+  repeated uint64 repeated_uint64 = 34;
-  repeated   sint32 repeated_sint32   = 35;
+  repeated sint32 repeated_sint32 = 35;
-  repeated   sint64 repeated_sint64   = 36;
+  repeated sint64 repeated_sint64 = 36;
-  repeated  fixed32 repeated_fixed32  = 37;
+  repeated fixed32 repeated_fixed32 = 37;
-  repeated  fixed64 repeated_fixed64  = 38;
+  repeated fixed64 repeated_fixed64 = 38;
   repeated sfixed32 repeated_sfixed32 = 39;
   repeated sfixed64 repeated_sfixed64 = 40;
-  repeated    float repeated_float    = 41;
+  repeated float repeated_float = 41;
-  repeated   double repeated_double   = 42;
+  repeated double repeated_double = 42;
-  repeated     bool repeated_bool     = 43;
+  repeated bool repeated_bool = 43;
-  repeated   string repeated_string   = 44;
+  repeated string repeated_string = 44;
-  repeated    bytes repeated_bytes    = 45;
+  repeated bytes repeated_bytes = 45;
 
   repeated ForeignMessage repeated_foreign_message = 49;
-  repeated Proto3Enum     repeated_foreign_enum    = 52;
+  repeated Proto3Enum repeated_foreign_enum = 52;
 
 
   oneof oneof_field {

js/test13.proto

@@ -47,24 +47,24 @@ enum TestEnum {
 
 message TestLooooooooooooooooooooooooooooooooooooooooooooooooooooooongName1 {
   optional TestLooooooooooooooooooooooooooooooooooooooooooooooooooooooongName2
-    a = 1;
+      a = 1;
   optional int32 b = 2;
 }
 
 message TestLooooooooooooooooooooooooooooooooooooooooooooooooooooooongName2 {
   optional TestLooooooooooooooooooooooooooooooooooooooooooooooooooooooongName3
-    a = 1;
+      a = 1;
   optional int32 b = 2;
 }
 
 message TestLooooooooooooooooooooooooooooooooooooooooooooooooooooooongName3 {
   optional TestLooooooooooooooooooooooooooooooooooooooooooooooooooooooongName4
-    a = 1;
+      a = 1;
   optional int32 b = 2;
 }
 
 message TestLooooooooooooooooooooooooooooooooooooooooooooooooooooooongName4 {
   optional TestLooooooooooooooooooooooooooooooooooooooooooooooooooooooongName1
-    a = 1;
+      a = 1;
   optional int32 b = 2;
 }

js/test15.proto

@@ -30,10 +30,10 @@
 
 syntax = "proto2";
 
-import "test13.proto";
-
 package jspb.filenametest.package1;
 
+import "test13.proto";
+
 extend TestMessage {
   optional int32 b = 2;
 }

js/test2.proto

@@ -30,13 +30,13 @@
 
 syntax = "proto2";
 
-option java_package = "com.google.apps.jspb.proto";
-option java_multiple_files = true;
-
 package jspb.test;
 
 import "test.proto";
 
+option java_package = "com.google.apps.jspb.proto";
+option java_multiple_files = true;
+
 message TestExtensionsMessage {
   optional int32 intfield = 1;
   extensions 100 to max;

js/test3.proto

@@ -30,11 +30,11 @@
 
 syntax = "proto2";
 
+package jspb.exttest;
+
 option java_package = "com.google.apps.jspb.proto";
 option java_multiple_files = true;
 
-package jspb.exttest;
-
 message TestExtensionsMessage {
   optional int32 intfield = 1;
   extensions 100 to max;

js/test4.proto

@@ -30,13 +30,13 @@
 
 syntax = "proto2";
 
-option java_package = "com.google.apps.jspb.proto";
-option java_multiple_files = true;
-
 package jspb.exttest;
 
 import "test3.proto";
 
+option java_package = "com.google.apps.jspb.proto";
+option java_multiple_files = true;
+
 extend TestExtensionsMessage {
   optional ExtensionMessage floating_msg_field_two = 103;
 }

js/test5.proto

@@ -30,11 +30,11 @@
 
 syntax = "proto2";
 
+package jspb.exttest.beta;
+
 option java_package = "com.google.apps.jspb.proto";
 option java_multiple_files = true;
 
-package jspb.exttest.beta;
-
 message TestBetaExtensionsMessage {
   extensions 100 to max;
 }

js/test8.proto

@@ -30,11 +30,11 @@
 
 syntax = "proto2";
 
+package jspb.exttest.nested;
+
 option java_package = "com.google.apps.jspb.proto";
 option java_multiple_files = true;
 
-package jspb.exttest.nested;
-
 message TestNestedExtensionsMessage {
   optional int32 intfield = 1;
   extensions 100 to max;

js/testbinary.proto

@@ -39,66 +39,66 @@ package jspb.test;
 // to ensure that the binary-format support will handle all field types
 // properly.
 message TestAllTypes {
-  optional    int32 optional_int32    =  1;
+  optional int32 optional_int32 = 1;
-  optional    int64 optional_int64    =  2;
+  optional int64 optional_int64 = 2;
-  optional   uint32 optional_uint32   =  3;
+  optional uint32 optional_uint32 = 3;
-  optional   uint64 optional_uint64   =  4;
+  optional uint64 optional_uint64 = 4;
-  optional   sint32 optional_sint32   =  5;
+  optional sint32 optional_sint32 = 5;
-  optional   sint64 optional_sint64   =  6;
+  optional sint64 optional_sint64 = 6;
-  optional  fixed32 optional_fixed32  =  7;
+  optional fixed32 optional_fixed32 = 7;
-  optional  fixed64 optional_fixed64  =  8;
+  optional fixed64 optional_fixed64 = 8;
-  optional sfixed32 optional_sfixed32 =  9;
+  optional sfixed32 optional_sfixed32 = 9;
   optional sfixed64 optional_sfixed64 = 10;
-  optional    float optional_float    = 11;
+  optional float optional_float = 11;
-  optional   double optional_double   = 12;
+  optional double optional_double = 12;
-  optional     bool optional_bool     = 13;
+  optional bool optional_bool = 13;
-  optional   string optional_string   = 14;
+  optional string optional_string = 14;
-  optional    bytes optional_bytes    = 15;
+  optional bytes optional_bytes = 15;
   optional group OptionalGroup = 16 {
     optional int32 a = 17;
   }
 
-  optional ForeignMessage                       optional_foreign_message = 19;
+  optional ForeignMessage optional_foreign_message = 19;
-  optional ForeignEnum                          optional_foreign_enum    = 22;
+  optional ForeignEnum optional_foreign_enum = 22;
 
   // Repeated
-  repeated    int32 repeated_int32    = 31;
+  repeated int32 repeated_int32 = 31;
-  repeated    int64 repeated_int64    = 32;
+  repeated int64 repeated_int64 = 32;
-  repeated   uint32 repeated_uint32   = 33;
+  repeated uint32 repeated_uint32 = 33;
-  repeated   uint64 repeated_uint64   = 34;
+  repeated uint64 repeated_uint64 = 34;
-  repeated   sint32 repeated_sint32   = 35;
+  repeated sint32 repeated_sint32 = 35;
-  repeated   sint64 repeated_sint64   = 36;
+  repeated sint64 repeated_sint64 = 36;
-  repeated  fixed32 repeated_fixed32  = 37;
+  repeated fixed32 repeated_fixed32 = 37;
-  repeated  fixed64 repeated_fixed64  = 38;
+  repeated fixed64 repeated_fixed64 = 38;
   repeated sfixed32 repeated_sfixed32 = 39;
   repeated sfixed64 repeated_sfixed64 = 40;
-  repeated    float repeated_float    = 41;
+  repeated float repeated_float = 41;
-  repeated   double repeated_double   = 42;
+  repeated double repeated_double = 42;
-  repeated     bool repeated_bool     = 43;
+  repeated bool repeated_bool = 43;
-  repeated   string repeated_string   = 44;
+  repeated string repeated_string = 44;
-  repeated    bytes repeated_bytes    = 45;
+  repeated bytes repeated_bytes = 45;
 
   repeated group RepeatedGroup = 46 {
     optional int32 a = 47;
   }
 
-  repeated ForeignMessage                       repeated_foreign_message = 49;
+  repeated ForeignMessage repeated_foreign_message = 49;
-  repeated ForeignEnum                          repeated_foreign_enum    = 52;
+  repeated ForeignEnum repeated_foreign_enum = 52;
 
   // Packed repeated
-  repeated    int32 packed_repeated_int32    = 61 [packed=true];
+  repeated int32 packed_repeated_int32 = 61 [packed = true];
-  repeated    int64 packed_repeated_int64    = 62 [packed=true];
+  repeated int64 packed_repeated_int64 = 62 [packed = true];
-  repeated   uint32 packed_repeated_uint32   = 63 [packed=true];
+  repeated uint32 packed_repeated_uint32 = 63 [packed = true];
-  repeated   uint64 packed_repeated_uint64   = 64 [packed=true];
+  repeated uint64 packed_repeated_uint64 = 64 [packed = true];
-  repeated   sint32 packed_repeated_sint32   = 65 [packed=true];
+  repeated sint32 packed_repeated_sint32 = 65 [packed = true];
-  repeated   sint64 packed_repeated_sint64   = 66 [packed=true];
+  repeated sint64 packed_repeated_sint64 = 66 [packed = true];
-  repeated  fixed32 packed_repeated_fixed32  = 67 [packed=true];
+  repeated fixed32 packed_repeated_fixed32 = 67 [packed = true];
-  repeated  fixed64 packed_repeated_fixed64  = 68 [packed=true];
+  repeated fixed64 packed_repeated_fixed64 = 68 [packed = true];
-  repeated sfixed32 packed_repeated_sfixed32 = 69 [packed=true];
+  repeated sfixed32 packed_repeated_sfixed32 = 69 [packed = true];
-  repeated sfixed64 packed_repeated_sfixed64 = 70 [packed=true];
+  repeated sfixed64 packed_repeated_sfixed64 = 70 [packed = true];
-  repeated    float packed_repeated_float    = 71 [packed=true];
+  repeated float packed_repeated_float = 71 [packed = true];
-  repeated   double packed_repeated_double   = 72 [packed=true];
+  repeated double packed_repeated_double = 72 [packed = true];
-  repeated     bool packed_repeated_bool     = 73 [packed=true];
+  repeated bool packed_repeated_bool = 73 [packed = true];
 
   oneof oneof_field {
     uint32 oneof_uint32 = 111;
@@ -132,55 +132,54 @@ message ExtendsWithMessage {
 }
 
 extend TestExtendable {
-  optional    int32 extend_optional_int32    =  1;
+  optional int32 extend_optional_int32 = 1;
-  optional    int64 extend_optional_int64    =  2;
+  optional int64 extend_optional_int64 = 2;
-  optional   uint32 extend_optional_uint32   =  3;
+  optional uint32 extend_optional_uint32 = 3;
-  optional   uint64 extend_optional_uint64   =  4;
+  optional uint64 extend_optional_uint64 = 4;
-  optional   sint32 extend_optional_sint32   =  5;
+  optional sint32 extend_optional_sint32 = 5;
-  optional   sint64 extend_optional_sint64   =  6;
+  optional sint64 extend_optional_sint64 = 6;
-  optional  fixed32 extend_optional_fixed32  =  7;
+  optional fixed32 extend_optional_fixed32 = 7;
-  optional  fixed64 extend_optional_fixed64  =  8;
+  optional fixed64 extend_optional_fixed64 = 8;
-  optional sfixed32 extend_optional_sfixed32 =  9;
+  optional sfixed32 extend_optional_sfixed32 = 9;
   optional sfixed64 extend_optional_sfixed64 = 10;
-  optional    float extend_optional_float    = 11;
+  optional float extend_optional_float = 11;
-  optional   double extend_optional_double   = 12;
+  optional double extend_optional_double = 12;
-  optional     bool extend_optional_bool     = 13;
+  optional bool extend_optional_bool = 13;
-  optional   string extend_optional_string   = 14;
+  optional string extend_optional_string = 14;
-  optional    bytes extend_optional_bytes    = 15;
+  optional bytes extend_optional_bytes = 15;
-  optional ForeignEnum extend_optional_foreign_enum    = 22;
+  optional ForeignEnum extend_optional_foreign_enum = 22;
-
+
-  repeated    int32 extend_repeated_int32    = 31;
+  repeated int32 extend_repeated_int32 = 31;
-  repeated    int64 extend_repeated_int64    = 32;
+  repeated int64 extend_repeated_int64 = 32;
-  repeated   uint32 extend_repeated_uint32   = 33;
+  repeated uint32 extend_repeated_uint32 = 33;
-  repeated   uint64 extend_repeated_uint64   = 34;
+  repeated uint64 extend_repeated_uint64 = 34;
-  repeated   sint32 extend_repeated_sint32   = 35;
+  repeated sint32 extend_repeated_sint32 = 35;
-  repeated   sint64 extend_repeated_sint64   = 36;
+  repeated sint64 extend_repeated_sint64 = 36;
-  repeated  fixed32 extend_repeated_fixed32  = 37;
+  repeated fixed32 extend_repeated_fixed32 = 37;
-  repeated  fixed64 extend_repeated_fixed64  = 38;
+  repeated fixed64 extend_repeated_fixed64 = 38;
   repeated sfixed32 extend_repeated_sfixed32 = 39;
   repeated sfixed64 extend_repeated_sfixed64 = 40;
-  repeated    float extend_repeated_float    = 41;
+  repeated float extend_repeated_float = 41;
-  repeated   double extend_repeated_double   = 42;
+  repeated double extend_repeated_double = 42;
-  repeated     bool extend_repeated_bool     = 43;
+  repeated bool extend_repeated_bool = 43;
-  repeated   string extend_repeated_string   = 44;
+  repeated string extend_repeated_string = 44;
-  repeated    bytes extend_repeated_bytes    = 45;
+  repeated bytes extend_repeated_bytes = 45;
-  repeated ForeignEnum extend_repeated_foreign_enum    = 52;
+  repeated ForeignEnum extend_repeated_foreign_enum = 52;
-
+
-  repeated    int32 extend_packed_repeated_int32    = 61 [packed=true];
+  repeated int32 extend_packed_repeated_int32 = 61 [packed = true];
-  repeated    int64 extend_packed_repeated_int64    = 62 [packed=true];
+  repeated int64 extend_packed_repeated_int64 = 62 [packed = true];
-  repeated   uint32 extend_packed_repeated_uint32   = 63 [packed=true];
+  repeated uint32 extend_packed_repeated_uint32 = 63 [packed = true];
-  repeated   uint64 extend_packed_repeated_uint64   = 64 [packed=true];
+  repeated uint64 extend_packed_repeated_uint64 = 64 [packed = true];
-  repeated   sint32 extend_packed_repeated_sint32   = 65 [packed=true];
+  repeated sint32 extend_packed_repeated_sint32 = 65 [packed = true];
-  repeated   sint64 extend_packed_repeated_sint64   = 66 [packed=true];
+  repeated sint64 extend_packed_repeated_sint64 = 66 [packed = true];
-  repeated  fixed32 extend_packed_repeated_fixed32  = 67 [packed=true];
+  repeated fixed32 extend_packed_repeated_fixed32 = 67 [packed = true];
-  repeated  fixed64 extend_packed_repeated_fixed64  = 68 [packed=true];
+  repeated fixed64 extend_packed_repeated_fixed64 = 68 [packed = true];
-  repeated sfixed32 extend_packed_repeated_sfixed32 = 69 [packed=true];
+  repeated sfixed32 extend_packed_repeated_sfixed32 = 69 [packed = true];
-  repeated sfixed64 extend_packed_repeated_sfixed64 = 70 [packed=true];
+  repeated sfixed64 extend_packed_repeated_sfixed64 = 70 [packed = true];
-  repeated    float extend_packed_repeated_float    = 71 [packed=true];
+  repeated float extend_packed_repeated_float = 71 [packed = true];
-  repeated   double extend_packed_repeated_double   = 72 [packed=true];
+  repeated double extend_packed_repeated_double = 72 [packed = true];
-  repeated     bool extend_packed_repeated_bool     = 73 [packed=true];
+  repeated bool extend_packed_repeated_bool = 73 [packed = true];
-  repeated ForeignEnum extend_packed_repeated_foreign_enum    = 82
+  repeated ForeignEnum extend_packed_repeated_foreign_enum = 82 [packed = true];
-      [packed=true];
 
 }
 
@@ -226,7 +225,7 @@ message MapEntryOptionalKeysBoolKey {
 
 message TestMapFieldsOptionalKeys {
   optional MapEntryOptionalKeysStringKey map_string_string = 1;
-  optional MapEntryOptionalKeysInt32Key map_int32_string= 8;
+  optional MapEntryOptionalKeysInt32Key map_int32_string = 8;
   optional MapEntryOptionalKeysInt64Key map_int64_string = 9;
   optional MapEntryOptionalKeysBoolKey map_bool_string = 10;
 }

js/testempty.proto

@@ -31,4 +31,3 @@
 syntax = "proto2";
 
 package javatests.com.google.apps.jspb;
-

python/google/protobuf/descriptor.py

@@ -855,7 +855,7 @@ class FileDescriptor(DescriptorBase):
   dependencies: List of other FileDescriptors this FileDescriptor depends on.
   public_dependencies: A list of FileDescriptors, subset of the dependencies
     above, which were declared as "public".
-  message_types_by_name: Dict of message names of their descriptors.
+  message_types_by_name: Dict of message names and their descriptors.
   enum_types_by_name: Dict of enum names and their descriptors.
   extensions_by_name: Dict of extension names and their descriptors.
   services_by_name: Dict of services names and their descriptors.

python/google/protobuf/internal/_parameterized.py

@@ -145,7 +145,6 @@ be wrapped into a tuple:
 
 __author__ = 'tmarek@google.com (Torsten Marek)'
 
-import collections
 import functools
 import re
 import types
@@ -157,6 +156,13 @@ import uuid
 
 import six
 
+try:
+  # Since python 3
+  import collections.abc as collections_abc
+except ImportError:
+  # Won't work after python 3.8
+  import collections as collections_abc
+
 ADDR_RE = re.compile(r'\<([a-zA-Z0-9_\-\.]+) object at 0x[a-fA-F0-9]+\>')
 _SEPARATOR = uuid.uuid1().hex
 _FIRST_ARG = object()
@@ -174,12 +180,12 @@ def _StrClass(cls):
 
 
 def _NonStringIterable(obj):
-  return (isinstance(obj, collections.Iterable) and not
+  return (isinstance(obj, collections_abc.Iterable) and not
           isinstance(obj, six.string_types))
 
 
 def _FormatParameterList(testcase_params):
-  if isinstance(testcase_params, collections.Mapping):
+  if isinstance(testcase_params, collections_abc.Mapping):
     return ', '.join('%s=%s' % (argname, _CleanRepr(value))
                      for argname, value in testcase_params.items())
   elif _NonStringIterable(testcase_params):
@@ -222,7 +228,7 @@ class _ParameterizedTestIter(object):
     def MakeBoundParamTest(testcase_params):
       @functools.wraps(test_method)
       def BoundParamTest(self):
-        if isinstance(testcase_params, collections.Mapping):
+        if isinstance(testcase_params, collections_abc.Mapping):
           test_method(self, **testcase_params)
         elif _NonStringIterable(testcase_params):
           test_method(self, *testcase_params)
@@ -291,7 +297,7 @@ def _ParameterDecorator(naming_type, testcases):
     if isinstance(obj, type):
       _ModifyClass(
           obj,
-          list(testcases) if not isinstance(testcases, collections.Sequence)
+          list(testcases) if not isinstance(testcases, collections_abc.Sequence)
           else testcases,
           naming_type)
       return obj

python/google/protobuf/internal/decoder.py

@@ -914,11 +914,11 @@ def _SkipGroup(buffer, pos, end):
     pos = new_pos
 
 
-def _DecodeGroup(buffer, pos):
+def _DecodeUnknownFieldSet(buffer, pos, end_pos=None):
-  """Decode group.  Returns the UnknownFieldSet and new position."""
+  """Decode UnknownFieldSet.  Returns the UnknownFieldSet and new position."""
 
   unknown_field_set = containers.UnknownFieldSet()
-  while 1:
+  while end_pos is None or pos < end_pos:
     (tag_bytes, pos) = ReadTag(buffer, pos)
     (tag, _) = _DecodeVarint(tag_bytes, 0)
     field_number, wire_type = wire_format.UnpackTag(tag)
@@ -945,7 +945,7 @@ def _DecodeUnknownField(buffer, pos, wire_type):
     data = buffer[pos:pos+size]
     pos += size
   elif wire_type == wire_format.WIRETYPE_START_GROUP:
-    (data, pos) = _DecodeGroup(buffer, pos)
+    (data, pos) = _DecodeUnknownFieldSet(buffer, pos)
   elif wire_type == wire_format.WIRETYPE_END_GROUP:
     return (0, -1)
   else:

python/google/protobuf/internal/descriptor_database_test.py

@@ -44,10 +44,11 @@ from google.protobuf import unittest_pb2
 from google.protobuf import descriptor_pb2
 from google.protobuf.internal import factory_test2_pb2
 from google.protobuf.internal import no_package_pb2
+from google.protobuf.internal import testing_refleaks
 from google.protobuf import descriptor_database
 
 
-class DescriptorDatabaseTest(unittest.TestCase):
+class DescriptorDatabaseTest(testing_refleaks.BaseTestCase):
 
   def testAdd(self):
     db = descriptor_database.DescriptorDatabase()

python/google/protobuf/internal/descriptor_pool_test.py

@@ -55,6 +55,7 @@ from google.protobuf.internal import factory_test2_pb2
 from google.protobuf.internal import file_options_test_pb2
 from google.protobuf.internal import more_messages_pb2
 from google.protobuf.internal import no_package_pb2
+from google.protobuf.internal import testing_refleaks
 from google.protobuf import descriptor
 from google.protobuf import descriptor_database
 from google.protobuf import descriptor_pool
@@ -560,7 +561,8 @@ class DescriptorPoolTestBase(object):
                       str(w[0].message))
 
 
-class DefaultDescriptorPoolTest(DescriptorPoolTestBase, unittest.TestCase):
+class DefaultDescriptorPoolTest(DescriptorPoolTestBase,
+                                testing_refleaks.BaseTestCase):
 
   def setUp(self):
     self.pool = descriptor_pool.Default()
@@ -595,7 +597,8 @@ class DefaultDescriptorPoolTest(DescriptorPoolTestBase, unittest.TestCase):
         unittest_pb2.DESCRIPTOR.services_by_name['TestService'])
 
 
-class CreateDescriptorPoolTest(DescriptorPoolTestBase, unittest.TestCase):
+class CreateDescriptorPoolTest(DescriptorPoolTestBase,
+                               testing_refleaks.BaseTestCase):
 
   def setUp(self):
     self.pool = descriptor_pool.DescriptorPool()
@@ -617,7 +620,7 @@ class CreateDescriptorPoolTest(DescriptorPoolTestBase, unittest.TestCase):
 
 
 class SecondaryDescriptorFromDescriptorDB(DescriptorPoolTestBase,
-                                          unittest.TestCase):
+                                          testing_refleaks.BaseTestCase):
 
   def setUp(self):
     self.factory_test1_fd = descriptor_pb2.FileDescriptorProto.FromString(
@@ -809,7 +812,7 @@ class ExtensionField(object):
     test.assertEqual(file_desc, field_desc.file)
 
 
-class AddDescriptorTest(unittest.TestCase):
+class AddDescriptorTest(testing_refleaks.BaseTestCase):
 
   def _TestMessage(self, prefix):
     pool = descriptor_pool.DescriptorPool()

python/google/protobuf/internal/message_factory_test.py

@@ -43,12 +43,13 @@ from google.protobuf import descriptor_pb2
 from google.protobuf.internal import api_implementation
 from google.protobuf.internal import factory_test1_pb2
 from google.protobuf.internal import factory_test2_pb2
+from google.protobuf.internal import testing_refleaks
 from google.protobuf import descriptor_database
 from google.protobuf import descriptor_pool
 from google.protobuf import message_factory
 
 
-class MessageFactoryTest(unittest.TestCase):
+class MessageFactoryTest(testing_refleaks.BaseTestCase):
 
   def setUp(self):
     self.factory_test1_fd = descriptor_pb2.FileDescriptorProto.FromString(

python/google/protobuf/internal/message_test.py

@@ -45,7 +45,6 @@ abstract interface.
 __author__ = 'gps@google.com (Gregory P. Smith)'
 
 
-import collections
 import copy
 import math
 import operator
@@ -55,6 +54,13 @@ import six
 import sys
 import warnings
 
+try:
+  # Since python 3
+  import collections.abc as collections_abc
+except ImportError:
+  # Won't work after python 3.8
+  import collections as collections_abc
+
 try:
   import unittest2 as unittest  # PY26
 except ImportError:
@@ -753,9 +759,9 @@ class MessageTest(BaseTestCase):
 
   def testRepeatedFieldsAreSequences(self, message_module):
     m = message_module.TestAllTypes()
-    self.assertIsInstance(m.repeated_int32, collections.MutableSequence)
+    self.assertIsInstance(m.repeated_int32, collections_abc.MutableSequence)
     self.assertIsInstance(m.repeated_nested_message,
-                          collections.MutableSequence)
+                          collections_abc.MutableSequence)
 
   def testRepeatedFieldsNotHashable(self, message_module):
     m = message_module.TestAllTypes()
@@ -2339,11 +2345,11 @@ class Proto3Test(BaseTestCase):
 
   def testMapsAreMapping(self):
     msg = map_unittest_pb2.TestMap()
-    self.assertIsInstance(msg.map_int32_int32, collections.Mapping)
+    self.assertIsInstance(msg.map_int32_int32, collections_abc.Mapping)
-    self.assertIsInstance(msg.map_int32_int32, collections.MutableMapping)
+    self.assertIsInstance(msg.map_int32_int32, collections_abc.MutableMapping)
-    self.assertIsInstance(msg.map_int32_foreign_message, collections.Mapping)
+    self.assertIsInstance(msg.map_int32_foreign_message, collections_abc.Mapping)
     self.assertIsInstance(msg.map_int32_foreign_message,
-                          collections.MutableMapping)
+                          collections_abc.MutableMapping)
 
   def testMapsCompare(self):
     msg = map_unittest_pb2.TestMap()

python/google/protobuf/internal/test_bad_identifiers.proto

@@ -43,10 +43,10 @@ message TestBadIdentifiers {
 // Make sure these reasonable extension names don't conflict with internal
 // variables.
 extend TestBadIdentifiers {
-  optional string message = 100 [default="foo"];
+  optional string message = 100 [default = "foo"];
-  optional string descriptor = 101 [default="bar"];
+  optional string descriptor = 101 [default = "bar"];
-  optional string reflection = 102 [default="baz"];
+  optional string reflection = 102 [default = "baz"];
-  optional string service = 103 [default="qux"];
+  optional string service = 103 [default = "qux"];
 }
 
 message AnotherMessage {}

python/google/protobuf/internal/text_format_test.py

@@ -798,6 +798,39 @@ class OnlyWorksWithProto2RightNowTests(TextFormatBase):
         self.RemoveRedundantZeros(text_format.MessageToString(message)),
         'text_format_unittest_data_oneof_implemented.txt')
 
+  def testPrintUnknownFields(self):
+    message = unittest_pb2.TestAllTypes()
+    message.optional_int32 = 101
+    message.optional_double = 102.0
+    message.optional_string = u'hello'
+    message.optional_bytes = b'103'
+    message.optionalgroup.a = 104
+    message.optional_nested_message.bb = 105
+    all_data = message.SerializeToString()
+    empty_message = unittest_pb2.TestEmptyMessage()
+    empty_message.ParseFromString(all_data)
+    self.assertEqual('1: 101\n'
+                     '12: 4636878028842991616\n'
+                     '14: "hello"\n'
+                     '15: "103"\n'
+                     '16 {\n'
+                     '  17: 104\n'
+                     '}\n'
+                     '18 {\n'
+                     '  1: 105\n'
+                     '}\n',
+                     text_format.MessageToString(empty_message,
+                                                 print_unknown_fields=True))
+    self.assertEqual('1: 101 '
+                     '12: 4636878028842991616 '
+                     '14: "hello" '
+                     '15: "103" '
+                     '16 { 17: 104 } '
+                     '18 { 1: 105 }',
+                     text_format.MessageToString(empty_message,
+                                                 print_unknown_fields=True,
+                                                 as_one_line=True))
+
   def testPrintInIndexOrder(self):
     message = unittest_pb2.TestFieldOrderings()
     # Fields are listed in index order instead of field number.

python/google/protobuf/internal/well_known_types.py

@@ -41,11 +41,17 @@ This files defines well known classes which need extra maintenance including:
 __author__ = 'jieluo@google.com (Jie Luo)'
 
 import calendar
-import collections
 from datetime import datetime
 from datetime import timedelta
 import six
 
+try:
+  # Since python 3
+  import collections.abc as collections_abc
+except ImportError:
+  # Won't work after python 3.8
+  import collections as collections_abc
+
 from google.protobuf.descriptor import FieldDescriptor
 
 _TIMESTAMPFOMAT = '%Y-%m-%dT%H:%M:%S'
@@ -88,6 +94,9 @@ class Any(object):
     return '/' in self.type_url and self.TypeName() == descriptor.full_name
 
 
+_EPOCH_DATETIME = datetime.utcfromtimestamp(0)
+
+
 class Timestamp(object):
   """Class for Timestamp message type."""
 
@@ -221,8 +230,9 @@ class Timestamp(object):
 
   def ToDatetime(self):
     """Converts Timestamp to datetime."""
-    return datetime.utcfromtimestamp(
+    return _EPOCH_DATETIME + timedelta(
-        self.seconds + self.nanos / float(_NANOS_PER_SECOND))
+        seconds=self.seconds, microseconds=_RoundTowardZero(
+            self.nanos, _NANOS_PER_MICROSECOND))
 
   def FromDatetime(self, dt):
     """Converts datetime to Timestamp."""
@@ -780,7 +790,7 @@ class Struct(object):
     for key, value in dictionary.items():
       _SetStructValue(self.fields[key], value)
 
-collections.MutableMapping.register(Struct)
+collections_abc.MutableMapping.register(Struct)
 
 
 class ListValue(object):
@@ -824,7 +834,7 @@ class ListValue(object):
     list_value.Clear()
     return list_value
 
-collections.MutableSequence.register(ListValue)
+collections_abc.MutableSequence.register(ListValue)
 
 
 WKTBASES = {

python/google/protobuf/internal/well_known_types_test.py

@@ -34,9 +34,15 @@
 
 __author__ = 'jieluo@google.com (Jie Luo)'
 
-import collections
 import datetime
 
+try:
+  # Since python 3
+  import collections.abc as collections_abc
+except ImportError:
+  # Won't work after python 3.8
+  import collections as collections_abc
+
 try:
   import unittest2 as unittest  #PY26
 except ImportError:
@@ -249,6 +255,14 @@ class TimeUtilTest(TimeUtilTestBase):
     self.assertEqual(datetime.datetime(1970, 1, 1, 0, 0, 1, 999000),
                      message.ToDatetime())
 
+    dt = datetime.datetime(2555, 2, 22, 1, 2, 3, 456789)
+    message.FromDatetime(dt)
+    self.assertEqual(dt, message.ToDatetime())
+
+    dt = datetime.datetime.max
+    message.FromDatetime(dt)
+    self.assertEqual(dt, message.ToDatetime())
+
   def testDatetimeConversionWithTimezone(self):
     class TZ(datetime.tzinfo):
 
@@ -740,7 +754,7 @@ class StructTest(unittest.TestCase):
 
   def testStruct(self):
     struct = struct_pb2.Struct()
-    self.assertIsInstance(struct, collections.Mapping)
+    self.assertIsInstance(struct, collections_abc.Mapping)
     self.assertEqual(0, len(struct))
     struct_class = struct.__class__
 
@@ -749,7 +763,7 @@ class StructTest(unittest.TestCase):
     struct['key3'] = True
     struct.get_or_create_struct('key4')['subkey'] = 11.0
     struct_list = struct.get_or_create_list('key5')
-    self.assertIsInstance(struct_list, collections.Sequence)
+    self.assertIsInstance(struct_list, collections_abc.Sequence)
     struct_list.extend([6, 'seven', True, False, None])
     struct_list.add_struct()['subkey2'] = 9
     struct['key6'] = {'subkey': {}}

python/google/protobuf/pyext/descriptor.cc

@@ -249,6 +249,7 @@ static PyObject* GetOrBuildOptions(const DescriptorClass *descriptor) {
   }
   ScopedPyObjectPtr value(
       PyEval_CallObject(message_class->AsPyObject(), NULL));
+  Py_DECREF(message_class);
   if (value == NULL) {
     return NULL;
   }
@@ -363,7 +364,7 @@ PyObject* NewInternedDescriptor(PyTypeObject* type,
     return it->second;
   }
   // Create a new descriptor object
-  PyBaseDescriptor* py_descriptor = PyObject_New(
+  PyBaseDescriptor* py_descriptor = PyObject_GC_New(
       PyBaseDescriptor, type);
   if (py_descriptor == NULL) {
     return NULL;
@@ -385,6 +386,8 @@ PyObject* NewInternedDescriptor(PyTypeObject* type,
   Py_INCREF(pool);
   py_descriptor->pool = pool;
 
+  PyObject_GC_Track(py_descriptor);
+
   if (was_created) {
     *was_created = true;
   }
@@ -398,41 +401,53 @@ static void Dealloc(PyBaseDescriptor* self) {
   Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
 }
 
+static int GcTraverse(PyObject* pself, visitproc visit, void* arg) {
+  PyBaseDescriptor* self = reinterpret_cast<PyBaseDescriptor*>(pself);
+  Py_VISIT(self->pool);
+  return 0;
+}
+
+static int GcClear(PyObject* pself) {
+  PyBaseDescriptor* self = reinterpret_cast<PyBaseDescriptor*>(pself);
+  Py_CLEAR(self->pool);
+  return 0;
+}
+
 static PyGetSetDef Getters[] = {
   {NULL}
 };
 
 PyTypeObject PyBaseDescriptor_Type = {
-  PyVarObject_HEAD_INIT(&PyType_Type, 0)
+    PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME
-  FULL_MODULE_NAME ".DescriptorBase",   // tp_name
+    ".DescriptorBase",                        // tp_name
-  sizeof(PyBaseDescriptor),             // tp_basicsize
+    sizeof(PyBaseDescriptor),                 // tp_basicsize
-  0,                                    // tp_itemsize
+    0,                                        // tp_itemsize
-  (destructor)Dealloc,                  // tp_dealloc
+    (destructor)Dealloc,                      // tp_dealloc
-  0,                                    // tp_print
+    0,                                        // tp_print
-  0,                                    // tp_getattr
+    0,                                        // tp_getattr
-  0,                                    // tp_setattr
+    0,                                        // tp_setattr
-  0,                                    // tp_compare
+    0,                                        // tp_compare
-  0,                                    // tp_repr
+    0,                                        // tp_repr
-  0,                                    // tp_as_number
+    0,                                        // tp_as_number
-  0,                                    // tp_as_sequence
+    0,                                        // tp_as_sequence
-  0,                                    // tp_as_mapping
+    0,                                        // tp_as_mapping
-  0,                                    // tp_hash
+    0,                                        // tp_hash
-  0,                                    // tp_call
+    0,                                        // tp_call
-  0,                                    // tp_str
+    0,                                        // tp_str
-  0,                                    // tp_getattro
+    0,                                        // tp_getattro
-  0,                                    // tp_setattro
+    0,                                        // tp_setattro
-  0,                                    // tp_as_buffer
+    0,                                        // tp_as_buffer
-  Py_TPFLAGS_DEFAULT,                   // tp_flags
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,  // tp_flags
-  "Descriptors base class",             // tp_doc
+    "Descriptors base class",                 // tp_doc
-  0,                                    // tp_traverse
+    GcTraverse,                               // tp_traverse
-  0,                                    // tp_clear
+    GcClear,                                  // tp_clear
-  0,                                    // tp_richcompare
+    0,                                        // tp_richcompare
-  0,                                    // tp_weaklistoffset
+    0,                                        // tp_weaklistoffset
-  0,                                    // tp_iter
+    0,                                        // tp_iter
-  0,                                    // tp_iternext
+    0,                                        // tp_iternext
-  0,                                    // tp_methods
+    0,                                        // tp_methods
-  0,                                    // tp_members
+    0,                                        // tp_members
-  Getters,                              // tp_getset
+    Getters,                                  // tp_getset
 };
 
 }  // namespace descriptor
@@ -1436,45 +1451,45 @@ static PyMethodDef Methods[] = {
 }  // namespace file_descriptor
 
 PyTypeObject PyFileDescriptor_Type = {
-  PyVarObject_HEAD_INIT(&PyType_Type, 0)
+    PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME
-  FULL_MODULE_NAME ".FileDescriptor",   // tp_name
+    ".FileDescriptor",                     // tp_name
-  sizeof(PyFileDescriptor),             // tp_basicsize
+    sizeof(PyFileDescriptor),              // tp_basicsize
-  0,                                    // tp_itemsize
+    0,                                     // tp_itemsize
-  (destructor)file_descriptor::Dealloc,  // tp_dealloc
+    (destructor)file_descriptor::Dealloc,  // tp_dealloc
-  0,                                    // tp_print
+    0,                                     // tp_print
-  0,                                    // tp_getattr
+    0,                                     // tp_getattr
-  0,                                    // tp_setattr
+    0,                                     // tp_setattr
-  0,                                    // tp_compare
+    0,                                     // tp_compare
-  0,                                    // tp_repr
+    0,                                     // tp_repr
-  0,                                    // tp_as_number
+    0,                                     // tp_as_number
-  0,                                    // tp_as_sequence
+    0,                                     // tp_as_sequence
-  0,                                    // tp_as_mapping
+    0,                                     // tp_as_mapping
-  0,                                    // tp_hash
+    0,                                     // tp_hash
-  0,                                    // tp_call
+    0,                                     // tp_call
-  0,                                    // tp_str
+    0,                                     // tp_str
-  0,                                    // tp_getattro
+    0,                                     // tp_getattro
-  0,                                    // tp_setattro
+    0,                                     // tp_setattro
-  0,                                    // tp_as_buffer
+    0,                                     // tp_as_buffer
-  Py_TPFLAGS_DEFAULT,                   // tp_flags
+    Py_TPFLAGS_DEFAULT,                    // tp_flags
-  "A File Descriptor",                  // tp_doc
+    "A File Descriptor",                   // tp_doc
-  0,                                    // tp_traverse
+    0,                                     // tp_traverse
-  0,                                    // tp_clear
+    0,                                     // tp_clear
-  0,                                    // tp_richcompare
+    0,                                     // tp_richcompare
-  0,                                    // tp_weaklistoffset
+    0,                                     // tp_weaklistoffset
-  0,                                    // tp_iter
+    0,                                     // tp_iter
-  0,                                    // tp_iternext
+    0,                                     // tp_iternext
-  file_descriptor::Methods,             // tp_methods
+    file_descriptor::Methods,              // tp_methods
-  0,                                    // tp_members
+    0,                                     // tp_members
-  file_descriptor::Getters,             // tp_getset
+    file_descriptor::Getters,              // tp_getset
-  &descriptor::PyBaseDescriptor_Type,   // tp_base
+    &descriptor::PyBaseDescriptor_Type,    // tp_base
-  0,                                    // tp_dict
+    0,                                     // tp_dict
-  0,                                    // tp_descr_get
+    0,                                     // tp_descr_get
-  0,                                    // tp_descr_set
+    0,                                     // tp_descr_set
-  0,                                    // tp_dictoffset
+    0,                                     // tp_dictoffset
-  0,                                    // tp_init
+    0,                                     // tp_init
-  0,                                    // tp_alloc
+    0,                                     // tp_alloc
-  0,                                    // tp_new
+    0,                                     // tp_new
-  PyObject_Del,                         // tp_free
+    PyObject_GC_Del,                       // tp_free
 };
 
 PyObject* PyFileDescriptor_FromDescriptor(

python/google/protobuf/pyext/descriptor_pool.cc

@@ -70,7 +70,7 @@ namespace cdescriptor_pool {
 // Create a Python DescriptorPool object, but does not fill the "pool"
 // attribute.
 static PyDescriptorPool* _CreateDescriptorPool() {
-  PyDescriptorPool* cpool = PyObject_New(
+  PyDescriptorPool* cpool = PyObject_GC_New(
       PyDescriptorPool, &PyDescriptorPool_Type);
   if (cpool == NULL) {
     return NULL;
@@ -88,6 +88,8 @@ static PyDescriptorPool* _CreateDescriptorPool() {
     return NULL;
   }
 
+  PyObject_GC_Track(cpool);
+
   return cpool;
 }
 
@@ -165,7 +167,19 @@ static void Dealloc(PyObject* pself) {
   delete self->descriptor_options;
   delete self->database;
   delete self->pool;
-  Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
+  Py_TYPE(self)->tp_free(pself);
+}
+
+static int GcTraverse(PyObject* pself, visitproc visit, void* arg) {
+  PyDescriptorPool* self = reinterpret_cast<PyDescriptorPool*>(pself);
+  Py_VISIT(self->py_message_factory);
+  return 0;
+}
+
+static int GcClear(PyObject* pself) {
+  PyDescriptorPool* self = reinterpret_cast<PyDescriptorPool*>(pself);
+  Py_CLEAR(self->py_message_factory);
+  return 0;
 }
 
 static PyObject* FindMessageByName(PyObject* self, PyObject* arg) {
@@ -629,45 +643,45 @@ static PyMethodDef Methods[] = {
 }  // namespace cdescriptor_pool
 
 PyTypeObject PyDescriptorPool_Type = {
-  PyVarObject_HEAD_INIT(&PyType_Type, 0)
+    PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME
-  FULL_MODULE_NAME ".DescriptorPool",  // tp_name
+    ".DescriptorPool",                        // tp_name
-  sizeof(PyDescriptorPool),            // tp_basicsize
+    sizeof(PyDescriptorPool),                 // tp_basicsize
-  0,                                   // tp_itemsize
+    0,                                        // tp_itemsize
-  cdescriptor_pool::Dealloc,           // tp_dealloc
+    cdescriptor_pool::Dealloc,                // tp_dealloc
-  0,                                   // tp_print
+    0,                                        // tp_print
-  0,                                   // tp_getattr
+    0,                                        // tp_getattr
-  0,                                   // tp_setattr
+    0,                                        // tp_setattr
-  0,                                   // tp_compare
+    0,                                        // tp_compare
-  0,                                   // tp_repr
+    0,                                        // tp_repr
-  0,                                   // tp_as_number
+    0,                                        // tp_as_number
-  0,                                   // tp_as_sequence
+    0,                                        // tp_as_sequence
-  0,                                   // tp_as_mapping
+    0,                                        // tp_as_mapping
-  0,                                   // tp_hash
+    0,                                        // tp_hash
-  0,                                   // tp_call
+    0,                                        // tp_call
-  0,                                   // tp_str
+    0,                                        // tp_str
-  0,                                   // tp_getattro
+    0,                                        // tp_getattro
-  0,                                   // tp_setattro
+    0,                                        // tp_setattro
-  0,                                   // tp_as_buffer
+    0,                                        // tp_as_buffer
-  Py_TPFLAGS_DEFAULT,                  // tp_flags
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,  // tp_flags
-  "A Descriptor Pool",                 // tp_doc
+    "A Descriptor Pool",                      // tp_doc
-  0,                                   // tp_traverse
+    cdescriptor_pool::GcTraverse,             // tp_traverse
-  0,                                   // tp_clear
+    cdescriptor_pool::GcClear,                // tp_clear
-  0,                                   // tp_richcompare
+    0,                                        // tp_richcompare
-  0,                                   // tp_weaklistoffset
+    0,                                        // tp_weaklistoffset
-  0,                                   // tp_iter
+    0,                                        // tp_iter
-  0,                                   // tp_iternext
+    0,                                        // tp_iternext
-  cdescriptor_pool::Methods,           // tp_methods
+    cdescriptor_pool::Methods,                // tp_methods
-  0,                                   // tp_members
+    0,                                        // tp_members
-  0,                                   // tp_getset
+    0,                                        // tp_getset
-  0,                                   // tp_base
+    0,                                        // tp_base
-  0,                                   // tp_dict
+    0,                                        // tp_dict
-  0,                                   // tp_descr_get
+    0,                                        // tp_descr_get
-  0,                                   // tp_descr_set
+    0,                                        // tp_descr_set
-  0,                                   // tp_dictoffset
+    0,                                        // tp_dictoffset
-  0,                                   // tp_init
+    0,                                        // tp_init
-  0,                                   // tp_alloc
+    0,                                        // tp_alloc
-  cdescriptor_pool::New,               // tp_new
+    cdescriptor_pool::New,                    // tp_new
-  PyObject_Del,                        // tp_free
+    PyObject_GC_Del,                          // tp_free
 };
 
 // This is the DescriptorPool which contains all the definitions from the

python/google/protobuf/pyext/descriptor_pool.h

@@ -50,8 +50,6 @@ struct CMessageClass;
 //
 // There is normally one pool per process. We make it a Python object only
 // because it contains many Python references.
-// TODO(amauryfa): See whether such objects can appear in reference cycles, and
-// consider adding support for the cyclic GC.
 //
 // "Methods" that interacts with this DescriptorPool are in the cdescriptor_pool
 // namespace.

python/google/protobuf/pyext/message.cc

@@ -310,12 +310,25 @@ static PyObject* New(PyTypeObject* type,
   return result.release();
 }
 
-static void Dealloc(CMessageClass *self) {
+static void Dealloc(PyObject* pself) {
+  CMessageClass* self = reinterpret_cast<CMessageClass*>(pself);
   Py_XDECREF(self->py_message_descriptor);
   Py_XDECREF(self->py_message_factory);
-  Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
+  return PyType_Type.tp_dealloc(pself);
 }
 
+static int GcTraverse(PyObject* pself, visitproc visit, void* arg) {
+  CMessageClass* self = reinterpret_cast<CMessageClass*>(pself);
+  Py_VISIT(self->py_message_descriptor);
+  Py_VISIT(self->py_message_factory);
+  return PyType_Type.tp_traverse(pself, visit, arg);
+}
+
+static int GcClear(PyObject* pself) {
+  // It's important to keep the descriptor and factory alive, until the
+  // C++ message is fully destructed.
+  return PyType_Type.tp_clear(pself);
+}
 
 // This function inserts and empty weakref at the end of the list of
 // subclasses for the main protocol buffer Message class.
@@ -328,11 +341,17 @@ static int InsertEmptyWeakref(PyTypeObject *base_type) {
   // hack addresses. For further background and the fix please see
   // https://bugs.python.org/issue17936.
   return 0;
+#else
+#ifdef Py_DEBUG
+  // The code below causes all new subclasses to append an entry, which is never
+  // cleared. This is a small memory leak, which we disable in Py_DEBUG mode
+  // to have stable refcounting checks.
 #else
   PyObject *subclasses = base_type->tp_subclasses;
   if (subclasses && PyList_CheckExact(subclasses)) {
     return PyList_Append(subclasses, kEmptyWeakref);
   }
+#endif  // !Py_DEBUG
   return 0;
 #endif  // PY_MAJOR_VERSION >= 3
 }
@@ -451,44 +470,44 @@ static PyObject* GetAttr(CMessageClass* self, PyObject* name) {
 }  // namespace message_meta
 
 static PyTypeObject _CMessageClass_Type = {
-  PyVarObject_HEAD_INIT(&PyType_Type, 0)
+    PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME
-  FULL_MODULE_NAME ".MessageMeta",     // tp_name
+    ".MessageMeta",                       // tp_name
-  sizeof(CMessageClass),               // tp_basicsize
+    sizeof(CMessageClass),                // tp_basicsize
-  0,                                   // tp_itemsize
+    0,                                    // tp_itemsize
-  (destructor)message_meta::Dealloc,   // tp_dealloc
+    message_meta::Dealloc,                // tp_dealloc
-  0,                                   // tp_print
+    0,                                    // tp_print
-  0,                                   // tp_getattr
+    0,                                    // tp_getattr
-  0,                                   // tp_setattr
+    0,                                    // tp_setattr
-  0,                                   // tp_compare
+    0,                                    // tp_compare
-  0,                                   // tp_repr
+    0,                                    // tp_repr
-  0,                                   // tp_as_number
+    0,                                    // tp_as_number
-  0,                                   // tp_as_sequence
+    0,                                    // tp_as_sequence
-  0,                                   // tp_as_mapping
+    0,                                    // tp_as_mapping
-  0,                                   // tp_hash
+    0,                                    // tp_hash
-  0,                                   // tp_call
+    0,                                    // tp_call
-  0,                                   // tp_str
+    0,                                    // tp_str
-  (getattrofunc)message_meta::GetAttr,  // tp_getattro
+    (getattrofunc)message_meta::GetAttr,  // tp_getattro
-  0,                                   // tp_setattro
+    0,                                    // tp_setattro
-  0,                                   // tp_as_buffer
+    0,                                    // tp_as_buffer
-  Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,  // tp_flags
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,  // tp_flags
-  "The metaclass of ProtocolMessages",  // tp_doc
+    "The metaclass of ProtocolMessages",                            // tp_doc
-  0,                                   // tp_traverse
+    message_meta::GcTraverse,  // tp_traverse
-  0,                                   // tp_clear
+    message_meta::GcClear,     // tp_clear
-  0,                                   // tp_richcompare
+    0,                         // tp_richcompare
-  0,                                   // tp_weaklistoffset
+    0,                         // tp_weaklistoffset
-  0,                                   // tp_iter
+    0,                         // tp_iter
-  0,                                   // tp_iternext
+    0,                         // tp_iternext
-  0,                                   // tp_methods
+    0,                         // tp_methods
-  0,                                   // tp_members
+    0,                         // tp_members
-  message_meta::Getters,               // tp_getset
+    message_meta::Getters,     // tp_getset
-  0,                                   // tp_base
+    0,                         // tp_base
-  0,                                   // tp_dict
+    0,                         // tp_dict
-  0,                                   // tp_descr_get
+    0,                         // tp_descr_get
-  0,                                   // tp_descr_set
+    0,                         // tp_descr_set
-  0,                                   // tp_dictoffset
+    0,                         // tp_dictoffset
-  0,                                   // tp_init
+    0,                         // tp_init
-  0,                                   // tp_alloc
+    0,                         // tp_alloc
-  message_meta::New,                   // tp_new
+    message_meta::New,         // tp_new
 };
 PyTypeObject* CMessageClass_Type = &_CMessageClass_Type;
 

python/google/protobuf/pyext/message.h

@@ -129,12 +129,13 @@ struct CMessageClass {
   const Descriptor* message_descriptor;
 
   // Owned reference, used to keep the pointer above alive.
+  // This reference must stay alive until all message pointers are destructed.
   PyObject* py_message_descriptor;
 
   // The Python MessageFactory used to create the class. It is needed to resolve
   // fields descriptors, including extensions fields; its C++ MessageFactory is
   // used to instantiate submessages.
-  // We own the reference, because it's important to keep the factory alive.
+  // This reference must stay alive until all message pointers are destructed.
   PyMessageFactory* py_message_factory;
 
   PyObject* AsPyObject() {

python/google/protobuf/pyext/message_factory.cc

@@ -69,9 +69,7 @@ PyMessageFactory* NewMessageFactory(PyTypeObject* type, PyDescriptorPool* pool)
   factory->message_factory = message_factory;
 
   factory->pool = pool;
-  // TODO(amauryfa): When the MessageFactory is not created from the
+  Py_INCREF(pool);
-  // DescriptorPool this reference should be owned, not borrowed.
-  // Py_INCREF(pool);
 
   factory->classes_by_descriptor = new PyMessageFactory::ClassesByMessageMap();
 
@@ -107,19 +105,37 @@ PyObject* New(PyTypeObject* type, PyObject* args, PyObject* kwargs) {
 static void Dealloc(PyObject* pself) {
   PyMessageFactory* self = reinterpret_cast<PyMessageFactory*>(pself);
 
-  // TODO(amauryfa): When the MessageFactory is not created from the
-  // DescriptorPool this reference should be owned, not borrowed.
-  // Py_CLEAR(self->pool);
   typedef PyMessageFactory::ClassesByMessageMap::iterator iterator;
   for (iterator it = self->classes_by_descriptor->begin();
        it != self->classes_by_descriptor->end(); ++it) {
-    Py_DECREF(it->second);
+    Py_CLEAR(it->second);
   }
   delete self->classes_by_descriptor;
   delete self->message_factory;
+  Py_CLEAR(self->pool);
   Py_TYPE(self)->tp_free(pself);
 }
 
+static int GcTraverse(PyObject* pself, visitproc visit, void* arg) {
+  PyMessageFactory* self = reinterpret_cast<PyMessageFactory*>(pself);
+  Py_VISIT(self->pool);
+  for (const auto& desc_and_class : *self->classes_by_descriptor) {
+    Py_VISIT(desc_and_class.second);
+  }
+  return 0;
+}
+
+static int GcClear(PyObject* pself) {
+  PyMessageFactory* self = reinterpret_cast<PyMessageFactory*>(pself);
+  // Here it's important to not clear self->pool, so that the C++ DescriptorPool
+  // is still alive when self->message_factory is destructed.
+  for (auto& desc_and_class : *self->classes_by_descriptor) {
+    Py_CLEAR(desc_and_class.second);
+  }
+
+  return 0;
+}
+
 // Add a message class to our database.
 int RegisterMessageClass(PyMessageFactory* self,
                          const Descriptor* message_descriptor,
@@ -234,44 +250,44 @@ static PyGetSetDef Getters[] = {
 
 PyTypeObject PyMessageFactory_Type = {
     PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME
-    ".MessageFactory",                        // tp_name
+    ".MessageFactory",         // tp_name
-    sizeof(PyMessageFactory),                 // tp_basicsize
+    sizeof(PyMessageFactory),  // tp_basicsize
-    0,                                        // tp_itemsize
+    0,                         // tp_itemsize
-    message_factory::Dealloc,                 // tp_dealloc
+    message_factory::Dealloc,  // tp_dealloc
-    0,                                        // tp_print
+    0,                         // tp_print
-    0,                                        // tp_getattr
+    0,                         // tp_getattr
-    0,                                        // tp_setattr
+    0,                         // tp_setattr
-    0,                                        // tp_compare
+    0,                         // tp_compare
-    0,                                        // tp_repr
+    0,                         // tp_repr
-    0,                                        // tp_as_number
+    0,                         // tp_as_number
-    0,                                        // tp_as_sequence
+    0,                         // tp_as_sequence
-    0,                                        // tp_as_mapping
+    0,                         // tp_as_mapping
-    0,                                        // tp_hash
+    0,                         // tp_hash
-    0,                                        // tp_call
+    0,                         // tp_call
-    0,                                        // tp_str
+    0,                         // tp_str
-    0,                                        // tp_getattro
+    0,                         // tp_getattro
-    0,                                        // tp_setattro
+    0,                         // tp_setattro
-    0,                                        // tp_as_buffer
+    0,                         // tp_as_buffer
-    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,  // tp_flags
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,  // tp_flags
-    "A static Message Factory",               // tp_doc
+    "A static Message Factory",                                     // tp_doc
-    0,                                        // tp_traverse
+    message_factory::GcTraverse,  // tp_traverse
-    0,                                        // tp_clear
+    message_factory::GcClear,     // tp_clear
-    0,                                        // tp_richcompare
+    0,                            // tp_richcompare
-    0,                                        // tp_weaklistoffset
+    0,                            // tp_weaklistoffset
-    0,                                        // tp_iter
+    0,                            // tp_iter
-    0,                                        // tp_iternext
+    0,                            // tp_iternext
-    message_factory::Methods,                 // tp_methods
+    message_factory::Methods,     // tp_methods
-    0,                                        // tp_members
+    0,                            // tp_members
-    message_factory::Getters,                 // tp_getset
+    message_factory::Getters,     // tp_getset
-    0,                                        // tp_base
+    0,                            // tp_base
-    0,                                        // tp_dict
+    0,                            // tp_dict
-    0,                                        // tp_descr_get
+    0,                            // tp_descr_get
-    0,                                        // tp_descr_set
+    0,                            // tp_descr_set
-    0,                                        // tp_dictoffset
+    0,                            // tp_dictoffset
-    0,                                        // tp_init
+    0,                            // tp_init
-    0,                                        // tp_alloc
+    0,                            // tp_alloc
-    message_factory::New,                     // tp_new
+    message_factory::New,         // tp_new
-    PyObject_Del,                             // tp_free
+    PyObject_GC_Del,              // tp_free
 };
 
 bool InitMessageFactory() {

python/google/protobuf/pyext/message_factory.h

@@ -57,9 +57,8 @@ struct PyMessageFactory {
   // The C++ one creates messages, when the Python one creates classes.
   MessageFactory* message_factory;
 
-  // borrowed reference to a Python DescriptorPool.
+  // Owned reference to a Python DescriptorPool.
-  // TODO(amauryfa): invert the dependency: the MessageFactory owns the
+  // This reference must stay until the message_factory is destructed.
-  // DescriptorPool, not the opposite.
   PyDescriptorPool* pool;
 
   // Make our own mapping to retrieve Python classes from C++ descriptors.

python/google/protobuf/pyext/proto2_api_test.proto

@@ -1,40 +0,0 @@
-// Protocol Buffers - Google's data interchange format
-// Copyright 2008 Google Inc.  All rights reserved.
-// https://developers.google.com/protocol-buffers/
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-//     * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-//     * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-//     * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-syntax = "proto2";
-
-package google.protobuf.python.internal;
-
-import "google/protobuf/internal/cpp/proto1_api_test.proto";
-
-message TestNestedProto1APIMessage {
-  optional int32 a = 1;
-  optional TestMessage.NestedMessage b = 2;
-}

python/google/protobuf/pyext/repeated_composite_container.cc

@@ -233,13 +233,12 @@ static PyObject* AddMessage(RepeatedCompositeContainer* self, PyObject* value) {
 static PyObject* AppendMethod(PyObject* pself, PyObject* value) {
   RepeatedCompositeContainer* self =
       reinterpret_cast<RepeatedCompositeContainer*>(pself);
-  PyObject* py_cmsg = AddMessage(self, value);
+  ScopedPyObjectPtr py_cmsg(AddMessage(self, value));
   if (py_cmsg == nullptr) {
     return nullptr;
   }
 
-  if (PyList_Append(self->child_messages, py_cmsg) < 0) {
+  if (PyList_Append(self->child_messages, py_cmsg.get()) < 0) {
-    Py_DECREF(py_cmsg);
     return nullptr;
   }
 
@@ -258,7 +257,7 @@ static PyObject* Insert(PyObject* pself, PyObject* args) {
     return nullptr;
   }
 
-  PyObject* py_cmsg = AddMessage(self, value);
+  ScopedPyObjectPtr py_cmsg(AddMessage(self, value));
   if (py_cmsg == nullptr) {
     return nullptr;
   }
@@ -277,7 +276,7 @@ static PyObject* Insert(PyObject* pself, PyObject* args) {
     }
   }
 
-  if (PyList_Insert(self->child_messages, index, py_cmsg) < 0) {
+  if (PyList_Insert(self->child_messages, index, py_cmsg.get()) < 0) {
     return nullptr;
   }
   Py_RETURN_NONE;

python/google/protobuf/text_format.py

@@ -51,6 +51,7 @@ if six.PY3:
   long = int  # pylint: disable=redefined-builtin,invalid-name
 
 # pylint: disable=g-import-not-at-top
+from google.protobuf.internal import decoder
 from google.protobuf.internal import type_checkers
 from google.protobuf import descriptor
 from google.protobuf import text_encoding
@@ -128,7 +129,8 @@ def MessageToString(message,
                     use_field_number=False,
                     descriptor_pool=None,
                     indent=0,
-                    message_formatter=None):
+                    message_formatter=None,
+                    print_unknown_fields=False):
   # type: (...) -> str
   """Convert protobuf message to text format.
 
@@ -159,6 +161,7 @@ def MessageToString(message,
     message_formatter: A function(message, indent, as_one_line): unicode|None
       to custom format selected sub-messages (usually based on message type).
       Use to pretty print parts of the protobuf for easier diffing.
+    print_unknown_fields: If True, unknown fields will be printed.
 
   Returns:
     A string of the text formatted protocol buffer message.
@@ -167,7 +170,8 @@ def MessageToString(message,
   printer = _Printer(out, indent, as_utf8, as_one_line,
                      use_short_repeated_primitives, pointy_brackets,
                      use_index_order, float_format, use_field_number,
-                     descriptor_pool, message_formatter)
+                     descriptor_pool, message_formatter,
+                     print_unknown_fields=print_unknown_fields)
   printer.PrintMessage(message)
   result = out.getvalue()
   out.close()
@@ -203,11 +207,19 @@ def PrintMessage(message,
                  float_format=None,
                  use_field_number=False,
                  descriptor_pool=None,
-                 message_formatter=None):
+                 message_formatter=None,
-  printer = _Printer(out, indent, as_utf8, as_one_line,
+                 print_unknown_fields=False):
-                     use_short_repeated_primitives, pointy_brackets,
+  printer = _Printer(
-                     use_index_order, float_format, use_field_number,
+      out=out, indent=indent, as_utf8=as_utf8,
-                     descriptor_pool, message_formatter)
+      as_one_line=as_one_line,
+      use_short_repeated_primitives=use_short_repeated_primitives,
+      pointy_brackets=pointy_brackets,
+      use_index_order=use_index_order,
+      float_format=float_format,
+      use_field_number=use_field_number,
+      descriptor_pool=descriptor_pool,
+      message_formatter=message_formatter,
+      print_unknown_fields=print_unknown_fields)
   printer.PrintMessage(message)
 
 
@@ -221,12 +233,14 @@ def PrintField(field,
                pointy_brackets=False,
                use_index_order=False,
                float_format=None,
-               message_formatter=None):
+               message_formatter=None,
+               print_unknown_fields=False):
   """Print a single field name/value pair."""
   printer = _Printer(out, indent, as_utf8, as_one_line,
                      use_short_repeated_primitives, pointy_brackets,
                      use_index_order, float_format,
-                     message_formatter=message_formatter)
+                     message_formatter=message_formatter,
+                     print_unknown_fields=print_unknown_fields)
   printer.PrintField(field, value)
 
 
@@ -240,12 +254,14 @@ def PrintFieldValue(field,
                     pointy_brackets=False,
                     use_index_order=False,
                     float_format=None,
-                    message_formatter=None):
+                    message_formatter=None,
+                    print_unknown_fields=False):
   """Print a single field value (not including name)."""
   printer = _Printer(out, indent, as_utf8, as_one_line,
                      use_short_repeated_primitives, pointy_brackets,
                      use_index_order, float_format,
-                     message_formatter=message_formatter)
+                     message_formatter=message_formatter,
+                     print_unknown_fields=print_unknown_fields)
   printer.PrintFieldValue(field, value)
 
 
@@ -274,6 +290,11 @@ def _BuildMessageFromTypeName(type_name, descriptor_pool):
   return message_type()
 
 
+# These values must match WireType enum in google/protobuf/wire_format.h.
+WIRETYPE_LENGTH_DELIMITED = 2
+WIRETYPE_START_GROUP = 3
+
+
 class _Printer(object):
   """Text format printer for protocol message."""
 
@@ -288,7 +309,8 @@ class _Printer(object):
                float_format=None,
                use_field_number=False,
                descriptor_pool=None,
-               message_formatter=None):
+               message_formatter=None,
+               print_unknown_fields=False):
     """Initialize the Printer.
 
     Floating point values can be formatted compactly with 15 digits of
@@ -317,6 +339,7 @@ class _Printer(object):
       message_formatter: A function(message, indent, as_one_line): unicode|None
         to custom format selected sub-messages (usually based on message type).
         Use to pretty print parts of the protobuf for easier diffing.
+      print_unknown_fields: If True, unknown fields will be printed.
     """
     self.out = out
     self.indent = indent
@@ -329,6 +352,7 @@ class _Printer(object):
     self.use_field_number = use_field_number
     self.descriptor_pool = descriptor_pool
     self.message_formatter = message_formatter
+    self.print_unknown_fields = print_unknown_fields
 
   def _TryPrintAsAnyMessage(self, message):
     """Serializes if message is a google.protobuf.Any field."""
@@ -392,6 +416,64 @@ class _Printer(object):
       else:
         self.PrintField(field, value)
 
+    if self.print_unknown_fields:
+      self._PrintUnknownFields(message.UnknownFields())
+
+  def _PrintUnknownFields(self, unknown_fields):
+    """Print unknown fields."""
+    out = self.out
+    for field in unknown_fields:
+      out.write(' ' * self.indent)
+      out.write(str(field.field_number))
+      if field.wire_type == WIRETYPE_START_GROUP:
+        if self.as_one_line:
+          out.write(' { ')
+        else:
+          out.write(' {\n')
+          self.indent += 2
+
+        self._PrintUnknownFields(field.data)
+
+        if self.as_one_line:
+          out.write('} ')
+        else:
+          out.write('}\n')
+          self.indent -= 2
+      elif field.wire_type == WIRETYPE_LENGTH_DELIMITED:
+        try:
+          # If this field is parseable as a Message, it is probably
+          # an embedded message.
+          # pylint: disable=protected-access
+          (embedded_unknown_message, pos) = decoder._DecodeUnknownFieldSet(
+              memoryview(field.data), 0, len(field.data))
+        except Exception:    # pylint: disable=broad-except
+          pos = 0
+
+        if pos == len(field.data):
+          if self.as_one_line:
+            out.write(' { ')
+          else:
+            out.write(' {\n')
+            self.indent += 2
+
+          self._PrintUnknownFields(embedded_unknown_message)
+
+          if self.as_one_line:
+            out.write('} ')
+          else:
+            out.write('}\n')
+            self.indent -= 2
+        else:
+          # A string or bytes field. self.as_utf8 may not work.
+          out.write(': \"')
+          out.write(text_encoding.CEscape(field.data, False))
+          out.write('\" ' if self.as_one_line else '\"\n')
+      else:
+        # varint, fixed32, fixed64
+        out.write(': ')
+        out.write(str(field.data))
+        out.write(' ' if self.as_one_line else '\n')
+
   def _PrintFieldName(self, field):
     """Print field name."""
     out = self.out

src/Makefile.am

@@ -131,7 +131,6 @@ nobase_include_HEADERS =                                         \
   google/protobuf/unknown_field_set.h                            \
   google/protobuf/wire_format.h                                  \
   google/protobuf/wire_format_lite.h                             \
-  google/protobuf/wire_format_lite_inl.h                         \
   google/protobuf/wrappers.pb.h                                  \
   google/protobuf/io/coded_stream.h                              \
   $(GZHEADERS)                                                   \

src/google/protobuf/compiler/cpp/cpp_enum.cc

@@ -72,6 +72,7 @@ EnumGenerator::EnumGenerator(const EnumDescriptor* descriptor,
   variables_["short_name"] = descriptor_->name();
   variables_["enumbase"] = options_.proto_h ? " : int" : "";
   variables_["nested_name"] = descriptor_->name();
+  variables_["resolved_name"] = ResolveKeyword(descriptor_->name());
   variables_["prefix"] =
       (descriptor_->containing_type() == NULL) ? "" : classname_ + "_";
 }
@@ -192,13 +193,13 @@ void EnumGenerator::GenerateGetEnumDescriptorSpecializations(
 
 void EnumGenerator::GenerateSymbolImports(io::Printer* printer) const {
   Formatter format(printer, variables_);
-  format("typedef $classname$ $nested_name$;\n");
+  format("typedef $classname$ $resolved_name$;\n");
 
   for (int j = 0; j < descriptor_->value_count(); j++) {
     std::string deprecated_attr = DeprecatedAttribute(
         options_, descriptor_->value(j)->options().deprecated());
     format(
-        "$1$static constexpr $nested_name$ ${2$$3$$}$ =\n"
+        "$1$static constexpr $resolved_name$ ${2$$3$$}$ =\n"
         "  $classname$_$3$;\n",
         deprecated_attr, descriptor_->value(j),
         EnumValueName(descriptor_->value(j)));
@@ -208,9 +209,9 @@ void EnumGenerator::GenerateSymbolImports(io::Printer* printer) const {
       "static inline bool $nested_name$_IsValid(int value) {\n"
       "  return $classname$_IsValid(value);\n"
       "}\n"
-      "static constexpr $nested_name$ ${1$$nested_name$_MIN$}$ =\n"
+      "static constexpr $resolved_name$ ${1$$nested_name$_MIN$}$ =\n"
       "  $classname$_$nested_name$_MIN;\n"
-      "static constexpr $nested_name$ ${1$$nested_name$_MAX$}$ =\n"
+      "static constexpr $resolved_name$ ${1$$nested_name$_MAX$}$ =\n"
       "  $classname$_$nested_name$_MAX;\n",
       descriptor_);
   if (generate_array_size_) {
@@ -231,7 +232,7 @@ void EnumGenerator::GenerateSymbolImports(io::Printer* printer) const {
       // version below.  Would this break our compatibility guarantees?
       format(
           "static inline const std::string& "
-          "$nested_name$_Name($nested_name$ value) {"
+          "$nested_name$_Name($resolved_name$ value) {"
           "\n"
           "  return $classname$_Name(value);\n"
           "}\n");
@@ -243,7 +244,7 @@ void EnumGenerator::GenerateSymbolImports(io::Printer* printer) const {
           "template<typename T>\n"
           "static inline const std::string& $nested_name$_Name(T enum_t_value) "
           "{\n"
-          "  static_assert(::std::is_same<T, $nested_name$>::value ||\n"
+          "  static_assert(::std::is_same<T, $resolved_name$>::value ||\n"
           "    ::std::is_integral<T>::value,\n"
           "    \"Incorrect type passed to function $nested_name$_Name.\");\n"
           "  return $classname$_Name(enum_t_value);\n"
@@ -251,7 +252,7 @@ void EnumGenerator::GenerateSymbolImports(io::Printer* printer) const {
     }
     format(
         "static inline bool $nested_name$_Parse(const std::string& name,\n"
-        "    $nested_name$* value) {\n"
+        "    $resolved_name$* value) {\n"
         "  return $classname$_Parse(name, value);\n"
         "}\n");
   }

src/google/protobuf/compiler/cpp/cpp_extension.cc

@@ -93,7 +93,7 @@ ExtensionGenerator::ExtensionGenerator(const FieldDescriptor* descriptor,
   variables_["extendee"] = ExtendeeClassName(descriptor_);
   variables_["type_traits"] = type_traits_;
   std::string name = descriptor_->name();
-  variables_["name"] = name;
+  variables_["name"] = ResolveKeyword(name);
   variables_["constant_name"] = FieldConstantName(descriptor_);
   variables_["field_type"] =
       StrCat(static_cast<int>(descriptor_->type()));
@@ -102,7 +102,7 @@ ExtensionGenerator::ExtensionGenerator(const FieldDescriptor* descriptor,
   std::string scope =
       IsScoped() ? ClassName(descriptor_->extension_scope(), false) + "::" : "";
   variables_["scope"] = scope;
-  std::string scoped_name = scope + name;
+  std::string scoped_name = scope + ResolveKeyword(name);
   variables_["scoped_name"] = scoped_name;
   variables_["number"] = StrCat(descriptor_->number());
 }

src/google/protobuf/compiler/cpp/cpp_helpers.cc

@@ -319,12 +319,12 @@ std::string ClassName(const Descriptor* descriptor) {
   if (parent) res += ClassName(parent) + "_";
   res += descriptor->name();
   if (IsMapEntryMessage(descriptor)) res += "_DoNotUse";
-  return res;
+  return ResolveKeyword(res);
 }
 
 std::string ClassName(const EnumDescriptor* enum_descriptor) {
   if (enum_descriptor->containing_type() == nullptr) {
-    return enum_descriptor->name();
+    return ResolveKeyword(enum_descriptor->name());
   } else {
     return ClassName(enum_descriptor->containing_type()) + "_" +
            enum_descriptor->name();
@@ -395,6 +395,13 @@ std::string SuperClassName(const Descriptor* descriptor,
                                                             : "::MessageLite");
 }
 
+std::string ResolveKeyword(const string& name) {
+  if (kKeywords.count(name) > 0) {
+    return name + "_";
+  }
+  return name;
+}
+
 std::string FieldName(const FieldDescriptor* field) {
   std::string result = field->name();
   LowerString(&result);
@@ -1345,8 +1352,6 @@ class ParseLoopGenerator {
     format_.Set("p_ns", "::" + ProtobufNamespace(options_));
     format_.Set("pi_ns", StrCat("::", ProtobufNamespace(options_), "::internal"));
     format_.Set("GOOGLE_PROTOBUF", MacroPrefix(options_));
-    format_.Set("kSlopBytes",
-                static_cast<int>(internal::ParseContext::kSlopBytes));
     std::map<std::string, std::string> vars;
     SetCommonVars(options_, &vars);
     format_.AddMap(vars);
@@ -1365,93 +1370,14 @@ class ParseLoopGenerator {
 
     format_(
         "const char* $classname$::_InternalParse(const char* ptr, "
-        "$pi_ns$::ParseContext* ctx) {\n"
+        "$pi_ns$::ParseContext* ctx) {\n");
-        "  $p_ns$::Arena* arena = GetArena(); (void)arena;\n"
-        "  while (!ctx->Done(&ptr)) {\n"
-        "    $uint32$ tag;\n"
-        "    ptr = $pi_ns$::ReadTag(ptr, &tag);\n"
-        "    $GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr);\n"
-        "    switch (tag >> 3) {\n");
-
     format_.Indent();
-    format_.Indent();
+    if (descriptor->file()->options().cc_enable_arenas()) {
-    format_.Indent();
+      format_("$p_ns$::Arena* arena = GetArenaNoVirtual(); (void)arena;\n");
-
-    for (const auto* field : ordered_fields) {
-      // Print the field's (or oneof's) proto-syntax definition as a comment.
-      // We don't want to print group bodies so we cut off after the first
-      // line.
-      std::string def;
-      {
-        DebugStringOptions options;
-        options.elide_group_body = true;
-        options.elide_oneof_body = true;
-        def = field->DebugStringWithOptions(options);
-        def = def.substr(0, def.find_first_of('\n'));
-      }
-      format_(
-          "// $1$\n"
-          "case $2$: {\n",
-          def, field->number());
-      format_.Indent();
-      GenerateCaseBody(field);
-      format_.Outdent();
-      format_("}\n");  // case
-    }                  // for fields
-
-    // Default case
-    format_("default: {\n");
-    if (!ordered_fields.empty()) format_("handle_unusual:\n");
-    format_(
-        "  if ((tag & 7) == 4 || tag == 0) {\n"
-        "    ctx->SetLastTag(tag);\n"
-        "    return ptr;\n"
-        "  }\n");
-    if (IsMapEntryMessage(descriptor)) {
-      format_("  break;\n");
-    } else {
-      if (descriptor->extension_range_count() > 0) {
-        format_("if (");
-        for (int i = 0; i < descriptor->extension_range_count(); i++) {
-          const Descriptor::ExtensionRange* range =
-              descriptor->extension_range(i);
-          if (i > 0) format_(" ||\n    ");
-
-          uint32 start_tag = WireFormatLite::MakeTag(
-              range->start, static_cast<WireFormatLite::WireType>(0));
-          uint32 end_tag = WireFormatLite::MakeTag(
-              range->end, static_cast<WireFormatLite::WireType>(0));
-
-          if (range->end > FieldDescriptor::kMaxNumber) {
-            format_("($1$u <= tag)", start_tag);
-          } else {
-            format_("($1$u <= tag && tag < $2$u)", start_tag, end_tag);
-          }
-        }
-        format_(") {\n");
-        format_(
-            "  ptr = _extensions_.ParseField(tag, ptr, \n"
-            "      internal_default_instance(), &_internal_metadata_, "
-            "ctx);\n"
-            "  $GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr != nullptr);\n"
-            "  break;\n"
-            "}\n");
-      }
-      format_(
-          "  ptr = UnknownFieldParse(tag,\n"
-          "    _internal_metadata_.mutable_unknown_fields(), ptr, ctx);\n"
-          "  $GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr != nullptr);\n"
-          "  break;\n");
     }
-    format_("}\n");  // default case
+    GenerateParseLoop(descriptor, ordered_fields);
-    format_.Outdent();
     format_.Outdent();
-    format_.Outdent();
+    format_("}\n");
-    format_(
-        "    }  // switch\n"
-        "  }  // while\n"
-        "  return ptr;\n"
-        "}\n");
   }
 
  private:
@@ -1469,18 +1395,25 @@ class ParseLoopGenerator {
               field_name.substr(2));  // remove ", "
       field_name = ", kFieldName";
     }
-    format_("if (arena != nullptr) {\n");
     if (HasFieldPresence(field->file())) {
-      format_("  HasBitSetters::set_has_$1$(this);\n", FieldName(field));
+      format_("HasBitSetters::set_has_$1$(this);\n", FieldName(field));
     }
+    string default_string =
+        field->default_value_string().empty()
+            ? "::" + ProtobufNamespace(options_) +
+                  "::internal::GetEmptyStringAlreadyInited()"
+            : QualifiedClassName(field->containing_type(), options_) +
+                  "::" + MakeDefaultName(field) + ".get()";
     format_(
+        "if (arena != nullptr) {\n"
         "  ptr = $pi_ns$::InlineCopyIntoArenaString$1$(&$2$_, ptr, ctx, "
         "  arena$3$);\n"
         "} else {\n"
-        "  ptr = $pi_ns$::InlineGreedyStringParser$1$($4$_$2$(), ptr, ctx$3$);"
+        "  ptr = "
+        "$pi_ns$::InlineGreedyStringParser$1$($2$_.MutableNoArenaNoDefault(&$4$"
+        "), ptr, ctx$3$);"
         "\n}\n",
-        utf8, FieldName(field), field_name,
+        utf8, FieldName(field), field_name, default_string);
-        field->is_repeated() ? "add" : "mutable");
   }
 
   void GenerateStrings(const FieldDescriptor* field, bool check_utf8) {
@@ -1766,6 +1699,93 @@ class ParseLoopGenerator {
       GenerateCaseBody(wiretype, field);
     }
   }
+
+  void GenerateParseLoop(
+      const Descriptor* descriptor,
+      const std::vector<const FieldDescriptor*>& ordered_fields) {
+    format_(
+        "while (!ctx->Done(&ptr)) {\n"
+        "  $uint32$ tag;\n"
+        "  ptr = $pi_ns$::ReadTag(ptr, &tag);\n"
+        "  $GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr);\n"
+        "  switch (tag >> 3) {\n");
+
+    format_.Indent();
+    format_.Indent();
+
+    for (const auto* field : ordered_fields) {
+      // Print the field's (or oneof's) proto-syntax definition as a comment.
+      // We don't want to print group bodies so we cut off after the first
+      // line.
+      std::string def;
+      {
+        DebugStringOptions options;
+        options.elide_group_body = true;
+        options.elide_oneof_body = true;
+        def = field->DebugStringWithOptions(options);
+        def = def.substr(0, def.find_first_of('\n'));
+      }
+      format_(
+          "// $1$\n"
+          "case $2$: {\n",
+          def, field->number());
+      format_.Indent();
+      GenerateCaseBody(field);
+      format_.Outdent();
+      format_("}\n");  // case
+    }                  // for fields
+
+    // Default case
+    format_("default: {\n");
+    if (!ordered_fields.empty()) format_("handle_unusual:\n");
+    format_(
+        "  if ((tag & 7) == 4 || tag == 0) {\n"
+        "    ctx->SetLastTag(tag);\n"
+        "    return ptr;\n"
+        "  }\n");
+    if (IsMapEntryMessage(descriptor)) {
+      format_("  break;\n");
+    } else {
+      if (descriptor->extension_range_count() > 0) {
+        format_("if (");
+        for (int i = 0; i < descriptor->extension_range_count(); i++) {
+          const Descriptor::ExtensionRange* range =
+              descriptor->extension_range(i);
+          if (i > 0) format_(" ||\n    ");
+
+          uint32 start_tag = WireFormatLite::MakeTag(
+              range->start, static_cast<WireFormatLite::WireType>(0));
+          uint32 end_tag = WireFormatLite::MakeTag(
+              range->end, static_cast<WireFormatLite::WireType>(0));
+
+          if (range->end > FieldDescriptor::kMaxNumber) {
+            format_("($1$u <= tag)", start_tag);
+          } else {
+            format_("($1$u <= tag && tag < $2$u)", start_tag, end_tag);
+          }
+        }
+        format_(") {\n");
+        format_(
+            "  ptr = _extensions_.ParseField(tag, ptr, \n"
+            "      internal_default_instance(), &_internal_metadata_, "
+            "ctx);\n"
+            "  $GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr != nullptr);\n"
+            "  break;\n"
+            "}\n");
+      }
+      format_(
+          "  ptr = UnknownFieldParse(tag, &_internal_metadata_, ptr, ctx);\n"
+          "  $GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr != nullptr);\n"
+          "  break;\n");
+    }
+    format_("}\n");  // default case
+    format_.Outdent();
+    format_.Outdent();
+    format_(
+        "  }  // switch\n"
+        "}  // while\n"
+        "return ptr;\n");
+  }
 };
 
 void GenerateParserLoop(const Descriptor* descriptor, const Options& options,

src/google/protobuf/compiler/cpp/cpp_helpers.h

@@ -143,6 +143,9 @@ std::string ReferenceFunctionName(const Descriptor* descriptor,
 std::string SuperClassName(const Descriptor* descriptor,
                            const Options& options);
 
+// Adds an underscore if necessary to prevent conflicting with a keyword.
+std::string ResolveKeyword(const string& name);
+
 // Get the (unqualified) name that should be used for this field in C++ code.
 // The name is coerced to lower-case to emulate proto1 behavior.  People
 // should be using lowercase-with-underscores style for proto field names

src/google/protobuf/compiler/cpp/cpp_message.cc

@@ -975,9 +975,11 @@ void MessageGenerator::GenerateClassDefinition(io::Printer* printer) {
         GOOGLE_CHECK(suffix == "UTF8Verify");
         format(
             "  bool ValidateKey() const {\n"
+            "#ifndef NDEBUG\n"
             "    ::$proto_ns$::internal::WireFormatLite::VerifyUtf8String(\n"
             "       key().data(), key().size(), ::$proto_ns$::internal::"
             "WireFormatLite::PARSE, \"$1$\");\n"
+            "#endif\n"
             "    return true;\n"
             " }\n",
             descriptor_->field(0)->full_name());
@@ -999,9 +1001,11 @@ void MessageGenerator::GenerateClassDefinition(io::Printer* printer) {
         GOOGLE_CHECK(suffix == "UTF8Verify");
         format(
             "  bool ValidateValue() const {\n"
+            "#ifndef NDEBUG\n"
             "    ::$proto_ns$::internal::WireFormatLite::VerifyUtf8String(\n"
             "       value().data(), value().size(), ::$proto_ns$::internal::"
             "WireFormatLite::PARSE, \"$1$\");\n"
+            "#endif\n"
             "    return true;\n"
             " }\n",
             descriptor_->field(1)->full_name());
@@ -1324,7 +1328,7 @@ void MessageGenerator::GenerateClassDefinition(io::Printer* printer) {
     const Descriptor* nested_type = descriptor_->nested_type(i);
     if (!IsMapEntryMessage(nested_type)) {
       format.Set("nested_full_name", ClassName(nested_type, false));
-      format.Set("nested_name", nested_type->name());
+      format.Set("nested_name", ResolveKeyword(nested_type->name()));
       format("typedef ${1$$nested_full_name$$}$ ${1$$nested_name$$}$;\n",
              nested_type);
     }

src/google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto

@@ -35,11 +35,13 @@
 // This file tests that various identifiers work as field and type names even
 // though the same identifiers are used internally by the C++ code generator.
 
+// LINT: LEGACY_NAMES
+
 syntax = "proto2";
 
 // Some generic_services option(s) added automatically.
 // See:  http://go/proto2-generic-services-default
-option cc_generic_services = true;     // auto-added
+option cc_generic_services = true; // auto-added
 
 // We don't put this in a package within proto2 because we need to make sure
 // that the generated code doesn't depend on being in the proto2 namespace.
@@ -55,17 +57,29 @@ message TestConflictingSymbolNames {
   optional int32 output = 2;
   optional string length = 3;
   repeated int32 i = 4;
-  repeated string new_element = 5 [ctype=STRING_PIECE];
+  repeated string new_element = 5 [ctype = STRING_PIECE];
   optional int32 total_size = 6;
   optional int32 tag = 7;
 
   enum TestEnum { FOO = 0; }
-  message Data1 { repeated int32 data = 1; }
+  message Data1 {
-  message Data2 { repeated TestEnum data = 1; }
+    repeated int32 data = 1;
-  message Data3 { repeated string data = 1; }
+  }
-  message Data4 { repeated Data4 data = 1; }
+  message Data2 {
-  message Data5 { repeated string data = 1 [ctype=STRING_PIECE]; }
+    repeated TestEnum data = 1;
-  message Data6 { repeated string data = 1 [ctype=CORD]; }
+  }
+  message Data3 {
+    repeated string data = 1;
+  }
+  message Data4 {
+    repeated Data4 data = 1;
+  }
+  message Data5 {
+    repeated string data = 1 [ctype = STRING_PIECE];
+  }
+  message Data6 {
+    repeated string data = 1 [ctype = CORD];
+  }
 
   optional int32 source = 8;
   optional int32 value = 9;
@@ -91,10 +105,10 @@ message TestConflictingSymbolNames {
   optional uint32 reflection = 27;
 
   message Cord {}
-  optional string some_cord = 28 [ctype=CORD];
+  optional string some_cord = 28 [ctype = CORD];
 
   message StringPiece {}
-  optional string some_string_piece = 29 [ctype=STRING_PIECE];
+  optional string some_string_piece = 29 [ctype = STRING_PIECE];
 
   // Some keywords.
   optional uint32 int = 30;
@@ -125,14 +139,15 @@ message TestConflictingSymbolNames {
   extensions 1000 to max;  // NO_PROTO3
 }
 
-message TestConflictingSymbolNamesExtension {                    // NO_PROTO3
+message TestConflictingSymbolNamesExtension {                      // NO_PROTO3
-  extend TestConflictingSymbolNames {                            // NO_PROTO3
+  extend TestConflictingSymbolNames {                              // NO_PROTO3
-    repeated int32 repeated_int32_ext = 20423638 [packed=true];  // NO_PROTO3
+    repeated int32 repeated_int32_ext = 20423638 [packed = true];  // NO_PROTO3
-  }                                                              // NO_PROTO3
+  }                                                                // NO_PROTO3
-}                                                                // NO_PROTO3
+}  // NO_PROTO3
 
 message TestConflictingEnumNames {  // NO_PROTO3
-  enum NestedConflictingEnum {      // NO_PROTO3
+  enum while {                      // NO_PROTO3
+    default = 0;                    // NO_PROTO3
     and = 1;                        // NO_PROTO3
     class = 2;                      // NO_PROTO3
     int = 3;                        // NO_PROTO3
@@ -140,18 +155,26 @@ message TestConflictingEnumNames {  // NO_PROTO3
     XOR = 5;                        // NO_PROTO3
   }                                 // NO_PROTO3
 
-  optional NestedConflictingEnum conflicting_enum = 1;  // NO_PROTO3
+  optional while conflicting_enum = 1;  // NO_PROTO3
 }  // NO_PROTO3
 
-enum ConflictingEnum {  // NO_PROTO3
+enum bool {             // NO_PROTO3
+  default = 0;          // NO_PROTO3
   NOT_EQ = 1;           // NO_PROTO3
   volatile = 2;         // NO_PROTO3
   return = 3;           // NO_PROTO3
-  NULL = 4;             // NO_PROTO3
 }  // NO_PROTO3
 
 message DummyMessage {}
 
+message NULL {
+  optional int32 int = 1;
+}
+
+extend TestConflictingSymbolNames {  // NO_PROTO3
+  optional int32 void = 314253;      // NO_PROTO3
+}                                    // NO_PROTO3
+
 // Message names that could conflict.
 message Shutdown {}
 message TableStruct {}

src/google/protobuf/compiler/cpp/cpp_unittest.cc

@@ -102,20 +102,33 @@ TEST(GENERATED_MESSAGE_TEST_NAME, TestConflictingSymbolNames) {
 
 TEST(GENERATED_MESSAGE_TEST_NAME, TestConflictingEnumNames) {
   protobuf_unittest::TestConflictingEnumNames message;
-  message.set_conflicting_enum(protobuf_unittest::TestConflictingEnumNames_NestedConflictingEnum_and_);
+  message.set_conflicting_enum(
+      protobuf_unittest::TestConflictingEnumNames_while_and_);
   EXPECT_EQ(1, message.conflicting_enum());
-  message.set_conflicting_enum(protobuf_unittest::TestConflictingEnumNames_NestedConflictingEnum_XOR);
+  message.set_conflicting_enum(
+      protobuf_unittest::TestConflictingEnumNames_while_XOR);
   EXPECT_EQ(5, message.conflicting_enum());
 
-  protobuf_unittest::ConflictingEnum conflicting_enum;
+  protobuf_unittest::bool_ conflicting_enum;
   conflicting_enum = protobuf_unittest::NOT_EQ;
   EXPECT_EQ(1, conflicting_enum);
   conflicting_enum = protobuf_unittest::return_;
   EXPECT_EQ(3, conflicting_enum);
-  conflicting_enum = protobuf_unittest::NULL_;
-  EXPECT_EQ(4, conflicting_enum);
 }
 
+TEST(GENERATED_MESSAGE_TEST_NAME, TestConflictingMessageNames) {
+  protobuf_unittest::NULL_ message;
+  message.set_int_(123);
+  EXPECT_EQ(message.int_(), 123);
+}
+
+TEST(GENERATED_MESSAGE_TEST_NAME, TestConflictingExtension) {
+  protobuf_unittest::TestConflictingSymbolNames message;
+  message.SetExtension(protobuf_unittest::void_, 123);
+  EXPECT_EQ(123, message.GetExtension(protobuf_unittest::void_));
+}
+
+
 }  // namespace cpp_unittest
 }  // namespace cpp
 }  // namespace compiler

src/google/protobuf/compiler/java/java_enum_field_lite.cc

@@ -53,6 +53,13 @@ namespace compiler {
 namespace java {
 
 namespace {
+bool EnableExperimentalRuntimeForLite() {
+#ifdef PROTOBUF_EXPERIMENT
+  return PROTOBUF_EXPERIMENT;
+#else   // PROTOBUF_EXPERIMENT
+  return false;
+#endif  // !PROTOBUF_EXPERIMENT
+}
 
 void SetEnumVariables(const FieldDescriptor* descriptor, int messageBitIndex,
                       int builderBitIndex, const FieldGeneratorInfo* info,
@@ -656,7 +663,7 @@ GenerateMembers(io::Printer* printer) const {
     printer->Annotate("{", "}", descriptor_);
   }
 
-  if (descriptor_->is_packed() &&
+  if (!EnableExperimentalRuntimeForLite() && descriptor_->is_packed() &&
       context_->HasGeneratedMethods(descriptor_->containing_type())) {
     printer->Print(variables_,
       "private int $name$MemoizedSerializedSize;\n");
@@ -929,6 +936,8 @@ GenerateParsingDoneCode(io::Printer* printer) const {
 
 void RepeatedImmutableEnumFieldLiteGenerator::
 GenerateSerializationCode(io::Printer* printer) const {
+  GOOGLE_CHECK(!EnableExperimentalRuntimeForLite());
+
   if (descriptor_->is_packed()) {
     printer->Print(variables_,
       "if (get$capitalized_name$List().size() > 0) {\n"
@@ -948,6 +957,8 @@ GenerateSerializationCode(io::Printer* printer) const {
 
 void RepeatedImmutableEnumFieldLiteGenerator::
 GenerateSerializedSizeCode(io::Printer* printer) const {
+  GOOGLE_CHECK(!EnableExperimentalRuntimeForLite());
+
   printer->Print(variables_,
     "{\n"
     "  int dataSize = 0;\n");

src/google/protobuf/compiler/java/java_helpers.cc

@@ -105,6 +105,7 @@ std::string FieldName(const FieldDescriptor* field) {
   return field_name;
 }
 
+
 }  // namespace
 
 void PrintGeneratedAnnotation(io::Printer* printer, char delimiter,

src/google/protobuf/compiler/java/java_primitive_field_lite.cc

@@ -56,6 +56,13 @@ using internal::WireFormat;
 using internal::WireFormatLite;
 
 namespace {
+bool EnableExperimentalRuntimeForLite() {
+#ifdef PROTOBUF_EXPERIMENT
+  return PROTOBUF_EXPERIMENT;
+#else   // PROTOBUF_EXPERIMENT
+  return false;
+#endif  // !PROTOBUF_EXPERIMENT
+}
 
 void SetPrimitiveVariables(const FieldDescriptor* descriptor,
                            int messageBitIndex, int builderBitIndex,
@@ -663,7 +670,7 @@ GenerateMembers(io::Printer* printer) const {
     "}\n");
   printer->Annotate("{", "}", descriptor_);
 
-  if (descriptor_->is_packed() &&
+  if (!EnableExperimentalRuntimeForLite() && descriptor_->is_packed() &&
       context_->HasGeneratedMethods(descriptor_->containing_type())) {
     printer->Print(variables_,
       "private int $name$MemoizedSerializedSize = -1;\n");
@@ -854,6 +861,8 @@ GenerateParsingDoneCode(io::Printer* printer) const {
 
 void RepeatedImmutablePrimitiveFieldLiteGenerator::
 GenerateSerializationCode(io::Printer* printer) const {
+  GOOGLE_CHECK(!EnableExperimentalRuntimeForLite());
+
   if (descriptor_->is_packed()) {
     // We invoke getSerializedSize in writeTo for messages that have packed
     // fields in ImmutableMessageGenerator::GenerateMessageSerializationMethods.
@@ -876,6 +885,8 @@ GenerateSerializationCode(io::Printer* printer) const {
 
 void RepeatedImmutablePrimitiveFieldLiteGenerator::
 GenerateSerializedSizeCode(io::Printer* printer) const {
+  GOOGLE_CHECK(!EnableExperimentalRuntimeForLite());
+
   printer->Print(variables_,
     "{\n"
     "  int dataSize = 0;\n");

src/google/protobuf/compiler/js/js_generator.cc

@@ -2426,9 +2426,13 @@ void Generator::GenerateClassFieldToObject(const GeneratorOptions& options,
     // all).  So we want to generate independent code.
     // The accessor for unset optional values without default should return
     // null. Those are converted to undefined in the generated object.
-    printer->Print("(f = ");
+    if (!use_default) {
+      printer->Print("(f = ");
+    }
     GenerateFieldValueExpression(printer, "msg", field, use_default);
-    printer->Print(") == null ? undefined : f");
+    if (!use_default) {
+      printer->Print(") == null ? undefined : f");
+    }
   }
 }
 

src/google/protobuf/compiler/parser_unittest.cc

@@ -62,8 +62,8 @@ namespace {
 
 class MockErrorCollector : public io::ErrorCollector {
  public:
-  MockErrorCollector() {}
+  MockErrorCollector() = default;
-  ~MockErrorCollector() {}
+  ~MockErrorCollector() override = default;
 
   std::string text_;
 
@@ -1114,17 +1114,17 @@ TEST_F(ParseMiscTest, ParsePackageWithSpaces) {
 
 TEST_F(ParseMiscTest, ParseFileOptions) {
   ExpectParsesTo(
-    "option java_package = \"com.google.foo\";\n"
+      "option java_package = \"com.google.foo\";\n"
-    "option optimize_for = CODE_SIZE;",
+      "option optimize_for = CODE_SIZE;",
-
+
-    "options {"
+      "options {"
-    "uninterpreted_option { name { name_part: \"java_package\" "
+      "uninterpreted_option { name { name_part: \"java_package\" "
-    "                              is_extension: false }"
+      "                              is_extension: false }"
-    "                       string_value: \"com.google.foo\"} "
+      "                       string_value: \"com.google.foo\"} "
-    "uninterpreted_option { name { name_part: \"optimize_for\" "
+      "uninterpreted_option { name { name_part: \"optimize_for\" "
-    "                              is_extension: false }"
+      "                              is_extension: false }"
-    "                       identifier_value: \"CODE_SIZE\" } "
+      "                       identifier_value: \"CODE_SIZE\" } "
-    "}");
+      "}");
 }
 
 // ===================================================================

src/google/protobuf/compiler/plugin.proto

@@ -45,6 +45,7 @@
 // flag "--${NAME}_out" is passed to protoc.
 
 syntax = "proto2";
+
 package google.protobuf.compiler;
 option java_package = "com.google.protobuf.compiler";
 option java_outer_classname = "PluginProtos";

src/google/protobuf/extension_set.cc

@@ -279,14 +279,14 @@ void ExtensionSet::ClearExtension(int number) {
 namespace {
 
 enum {
-  REPEATED,
+  REPEATED_FIELD,
-  OPTIONAL
+  OPTIONAL_FIELD
 };
 
 }  // namespace
 
-#define GOOGLE_DCHECK_TYPE(EXTENSION, LABEL, CPPTYPE)                             \
+#define GOOGLE_DCHECK_TYPE(EXTENSION, LABEL, CPPTYPE)                                 \
-  GOOGLE_DCHECK_EQ((EXTENSION).is_repeated ? REPEATED : OPTIONAL, LABEL);         \
+  GOOGLE_DCHECK_EQ((EXTENSION).is_repeated ? REPEATED_FIELD : OPTIONAL_FIELD, LABEL); \
   GOOGLE_DCHECK_EQ(cpp_type((EXTENSION).type), WireFormatLite::CPPTYPE_##CPPTYPE)
 
 // -------------------------------------------------------------------
@@ -300,7 +300,7 @@ LOWERCASE ExtensionSet::Get##CAMELCASE(int number,                             \
   if (extension == NULL || extension->is_cleared) {                            \
     return default_value;                                                      \
   } else {                                                                     \
-    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, UPPERCASE);                              \
+    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, UPPERCASE);                        \
     return extension->LOWERCASE##_value;                                       \
   }                                                                            \
 }                                                                              \
@@ -314,7 +314,7 @@ void ExtensionSet::Set##CAMELCASE(int number, FieldType type,                  \
     GOOGLE_DCHECK_EQ(cpp_type(extension->type), WireFormatLite::CPPTYPE_##UPPERCASE); \
     extension->is_repeated = false;                                            \
   } else {                                                                     \
-    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, UPPERCASE);                              \
+    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, UPPERCASE);                        \
   }                                                                            \
   extension->is_cleared = false;                                               \
   extension->LOWERCASE##_value = value;                                        \
@@ -323,7 +323,7 @@ void ExtensionSet::Set##CAMELCASE(int number, FieldType type,                  \
 LOWERCASE ExtensionSet::GetRepeated##CAMELCASE(int number, int index) const {  \
   const Extension* extension = FindOrNull(number);                             \
   GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty).";         \
-  GOOGLE_DCHECK_TYPE(*extension, REPEATED, UPPERCASE);                                \
+  GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, UPPERCASE);                          \
   return extension->repeated_##LOWERCASE##_value->Get(index);                  \
 }                                                                              \
                                                                                \
@@ -331,7 +331,7 @@ void ExtensionSet::SetRepeated##CAMELCASE(                                     \
     int number, int index, LOWERCASE value) {                                  \
   Extension* extension = FindOrNull(number);                                   \
   GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty).";         \
-  GOOGLE_DCHECK_TYPE(*extension, REPEATED, UPPERCASE);                                \
+  GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, UPPERCASE);                          \
   extension->repeated_##LOWERCASE##_value->Set(index, value);                  \
 }                                                                              \
                                                                                \
@@ -347,7 +347,7 @@ void ExtensionSet::Add##CAMELCASE(int number, FieldType type,                  \
     extension->repeated_##LOWERCASE##_value =                                  \
       Arena::CreateMessage<RepeatedField<LOWERCASE> >(arena_);                 \
   } else {                                                                     \
-    GOOGLE_DCHECK_TYPE(*extension, REPEATED, UPPERCASE);                              \
+    GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, UPPERCASE);                        \
     GOOGLE_DCHECK_EQ(extension->is_packed, packed);                                   \
   }                                                                            \
   extension->repeated_##LOWERCASE##_value->Add(value);                         \
@@ -456,7 +456,7 @@ int ExtensionSet::GetEnum(int number, int default_value) const {
     // Not present.  Return the default value.
     return default_value;
   } else {
-    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, ENUM);
+    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, ENUM);
     return extension->enum_value;
   }
 }
@@ -469,7 +469,7 @@ void ExtensionSet::SetEnum(int number, FieldType type, int value,
     GOOGLE_DCHECK_EQ(cpp_type(extension->type), WireFormatLite::CPPTYPE_ENUM);
     extension->is_repeated = false;
   } else {
-    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, ENUM);
+    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, ENUM);
   }
   extension->is_cleared = false;
   extension->enum_value = value;
@@ -478,14 +478,14 @@ void ExtensionSet::SetEnum(int number, FieldType type, int value,
 int ExtensionSet::GetRepeatedEnum(int number, int index) const {
   const Extension* extension = FindOrNull(number);
   GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty).";
-  GOOGLE_DCHECK_TYPE(*extension, REPEATED, ENUM);
+  GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, ENUM);
   return extension->repeated_enum_value->Get(index);
 }
 
 void ExtensionSet::SetRepeatedEnum(int number, int index, int value) {
   Extension* extension = FindOrNull(number);
   GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty).";
-  GOOGLE_DCHECK_TYPE(*extension, REPEATED, ENUM);
+  GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, ENUM);
   extension->repeated_enum_value->Set(index, value);
 }
 
@@ -501,7 +501,7 @@ void ExtensionSet::AddEnum(int number, FieldType type,
     extension->repeated_enum_value =
         Arena::CreateMessage<RepeatedField<int> >(arena_);
   } else {
-    GOOGLE_DCHECK_TYPE(*extension, REPEATED, ENUM);
+    GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, ENUM);
     GOOGLE_DCHECK_EQ(extension->is_packed, packed);
   }
   extension->repeated_enum_value->Add(value);
@@ -517,7 +517,7 @@ const std::string& ExtensionSet::GetString(
     // Not present.  Return the default value.
     return default_value;
   } else {
-    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, STRING);
+    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, STRING);
     return *extension->string_value;
   }
 }
@@ -531,7 +531,7 @@ std::string* ExtensionSet::MutableString(int number, FieldType type,
     extension->is_repeated = false;
     extension->string_value = Arena::Create<std::string>(arena_);
   } else {
-    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, STRING);
+    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, STRING);
   }
   extension->is_cleared = false;
   return extension->string_value;
@@ -541,14 +541,14 @@ const std::string& ExtensionSet::GetRepeatedString(int number,
                                                    int index) const {
   const Extension* extension = FindOrNull(number);
   GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty).";
-  GOOGLE_DCHECK_TYPE(*extension, REPEATED, STRING);
+  GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, STRING);
   return extension->repeated_string_value->Get(index);
 }
 
 std::string* ExtensionSet::MutableRepeatedString(int number, int index) {
   Extension* extension = FindOrNull(number);
   GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty).";
-  GOOGLE_DCHECK_TYPE(*extension, REPEATED, STRING);
+  GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, STRING);
   return extension->repeated_string_value->Mutable(index);
 }
 
@@ -563,7 +563,7 @@ std::string* ExtensionSet::AddString(int number, FieldType type,
     extension->repeated_string_value =
         Arena::CreateMessage<RepeatedPtrField<std::string>>(arena_);
   } else {
-    GOOGLE_DCHECK_TYPE(*extension, REPEATED, STRING);
+    GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, STRING);
   }
   return extension->repeated_string_value->Add();
 }
@@ -578,7 +578,7 @@ const MessageLite& ExtensionSet::GetMessage(
     // Not present.  Return the default value.
     return default_value;
   } else {
-    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE);
+    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, MESSAGE);
     if (extension->is_lazy) {
       return extension->lazymessage_value->GetMessage(default_value);
     } else {
@@ -605,7 +605,7 @@ MessageLite* ExtensionSet::MutableMessage(int number, FieldType type,
     extension->is_cleared = false;
     return extension->message_value;
   } else {
-    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE);
+    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, MESSAGE);
     extension->is_cleared = false;
     if (extension->is_lazy) {
       return extension->lazymessage_value->MutableMessage(prototype);
@@ -644,7 +644,7 @@ void ExtensionSet::SetAllocatedMessage(int number, FieldType type,
       extension->message_value->CheckTypeAndMergeFrom(*message);
     }
   } else {
-    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE);
+    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, MESSAGE);
     if (extension->is_lazy) {
       extension->lazymessage_value->SetAllocatedMessage(message);
     } else {
@@ -680,7 +680,7 @@ void ExtensionSet::UnsafeArenaSetAllocatedMessage(
     extension->is_lazy = false;
     extension->message_value = message;
   } else {
-    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE);
+    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, MESSAGE);
     if (extension->is_lazy) {
       extension->lazymessage_value->UnsafeArenaSetAllocatedMessage(message);
     } else {
@@ -700,7 +700,7 @@ MessageLite* ExtensionSet::ReleaseMessage(int number,
     // Not present.  Return NULL.
     return NULL;
   } else {
-    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE);
+    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, MESSAGE);
     MessageLite* ret = NULL;
     if (extension->is_lazy) {
       ret = extension->lazymessage_value->ReleaseMessage(prototype);
@@ -729,7 +729,7 @@ MessageLite* ExtensionSet::UnsafeArenaReleaseMessage(
     // Not present.  Return NULL.
     return NULL;
   } else {
-    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE);
+    GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, MESSAGE);
     MessageLite* ret = NULL;
     if (extension->is_lazy) {
       ret = extension->lazymessage_value->UnsafeArenaReleaseMessage(prototype);
@@ -752,14 +752,14 @@ const MessageLite& ExtensionSet::GetRepeatedMessage(
     int number, int index) const {
   const Extension* extension = FindOrNull(number);
   GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty).";
-  GOOGLE_DCHECK_TYPE(*extension, REPEATED, MESSAGE);
+  GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, MESSAGE);
   return extension->repeated_message_value->Get(index);
 }
 
 MessageLite* ExtensionSet::MutableRepeatedMessage(int number, int index) {
   Extension* extension = FindOrNull(number);
   GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty).";
-  GOOGLE_DCHECK_TYPE(*extension, REPEATED, MESSAGE);
+  GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, MESSAGE);
   return extension->repeated_message_value->Mutable(index);
 }
 
@@ -774,7 +774,7 @@ MessageLite* ExtensionSet::AddMessage(int number, FieldType type,
     extension->repeated_message_value =
         Arena::CreateMessage<RepeatedPtrField<MessageLite> >(arena_);
   } else {
-    GOOGLE_DCHECK_TYPE(*extension, REPEATED, MESSAGE);
+    GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, MESSAGE);
   }
 
   // RepeatedPtrField<MessageLite> does not know how to Add() since it cannot

src/google/protobuf/extension_set_inl.h

@@ -235,11 +235,13 @@ const char* ExtensionSet::ParseMessageSetItemTmpl(const char* ptr,
                                    extension.descriptor);
 
           const char* p;
+          // We can't use regular parse from string as we have to track
+          // proper recursion depth and descriptor pools.
           ParseContext tmp_ctx(ctx->depth(), false, &p, payload);
           tmp_ctx.data().pool = ctx->data().pool;
           tmp_ctx.data().factory = ctx->data().factory;
-          GOOGLE_PROTOBUF_PARSER_ASSERT(
+          GOOGLE_PROTOBUF_PARSER_ASSERT(value->_InternalParse(p, &tmp_ctx) &&
-              tmp_ctx.AtLegitimateEnd(value->_InternalParse(p, &tmp_ctx)));
+                                         tmp_ctx.EndedAtLimit());
         }
         type_id = 0;
       }

src/google/protobuf/generated_message_reflection.cc

@@ -50,6 +50,8 @@
 #include <google/protobuf/wire_format.h>
 
 
+#include <google/protobuf/port_def.inc>
+
 #define GOOGLE_PROTOBUF_HAS_ONEOF
 
 namespace google {

src/google/protobuf/generated_message_table_driven.h

@@ -273,23 +273,24 @@ inline uint8* TableSerializeToArray(const MessageLite& msg,
 
 template <typename T>
 struct CompareHelper {
-  bool operator()(const T& a, const T& b) { return a < b; }
+  bool operator()(const T& a, const T& b) const { return a < b; }
 };
 
 template <>
 struct CompareHelper<ArenaStringPtr> {
-  bool operator()(const ArenaStringPtr& a, const ArenaStringPtr& b) {
+  bool operator()(const ArenaStringPtr& a, const ArenaStringPtr& b) const {
     return a.Get() < b.Get();
   }
 };
 
 struct CompareMapKey {
   template <typename T>
-  bool operator()(const MapEntryHelper<T>& a, const MapEntryHelper<T>& b) {
+  bool operator()(const MapEntryHelper<T>& a,
+                  const MapEntryHelper<T>& b) const {
     return Compare(a.key_, b.key_);
   }
   template <typename T>
-  bool Compare(const T& a, const T& b) {
+  bool Compare(const T& a, const T& b) const {
     return CompareHelper<T>()(a, b);
   }
 };

src/google/protobuf/generated_message_util.cc

@@ -35,9 +35,13 @@
 #include <google/protobuf/generated_message_util.h>
 
 #include <limits>
+
+#ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP
 // We're only using this as a standard way for getting the thread id.
 // We're not using any thread functionality.
 #include <thread>  // NOLINT
+#endif  // #ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP
+
 #include <vector>
 
 #include <google/protobuf/io/coded_stream_inl.h>
@@ -784,8 +788,17 @@ void InitSCCImpl(SCCInfoBase* scc) {
   static WrappedMutex mu{GOOGLE_PROTOBUF_LINKER_INITIALIZED};
   // Either the default in case no initialization is running or the id of the
   // thread that is currently initializing.
+#ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP
   static std::atomic<std::thread::id> runner;
   auto me = std::this_thread::get_id();
+#else
+  // This is a lightweight replacement for std::thread::id. std::thread does not
+  // work on Windows XP SP2 with the latest VC++ libraries, because it utilizes
+  // the Concurrency Runtime that is only supported on Windows XP SP3 and above.
+  static std::atomic_llong runner(-1);
+  auto me = ::GetCurrentThreadId();
+#endif  // #ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP
+
   // This will only happen because the constructor will call InitSCC while
   // constructing the default instance.
   if (runner.load(std::memory_order_relaxed) == me) {
@@ -799,7 +812,13 @@ void InitSCCImpl(SCCInfoBase* scc) {
   mu.Lock();
   runner.store(me, std::memory_order_relaxed);
   InitSCC_DFS(scc);
+
+#ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP
   runner.store(std::thread::id{}, std::memory_order_relaxed);
+#else
+  runner.store(-1, std::memory_order_relaxed);
+#endif  // #ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP
+
   mu.Unlock();
 }
 

src/google/protobuf/implicit_weak_message.cc

@@ -41,17 +41,18 @@ namespace google {
 namespace protobuf {
 namespace internal {
 
-bool ImplicitWeakMessage::MergePartialFromCodedStream(io::CodedInputStream* input) {
-  io::StringOutputStream string_stream(&data_);
-  io::CodedOutputStream coded_stream(&string_stream, false);
-  return WireFormatLite::SkipMessage(input, &coded_stream);
-}
-
 #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
 const char* ImplicitWeakMessage::_InternalParse(const char* ptr,
                                                 ParseContext* ctx) {
   return ctx->AppendString(ptr, &data_);
 }
+#else
+bool ImplicitWeakMessage::MergePartialFromCodedStream(
+    io::CodedInputStream* input) {
+  io::StringOutputStream string_stream(&data_);
+  io::CodedOutputStream coded_stream(&string_stream, false);
+  return WireFormatLite::SkipMessage(input, &coded_stream);
+}
 #endif
 
 ExplicitlyConstructed<ImplicitWeakMessage>

src/google/protobuf/implicit_weak_message.h

@@ -76,10 +76,10 @@ class PROTOBUF_EXPORT ImplicitWeakMessage : public MessageLite {
     data_.append(static_cast<const ImplicitWeakMessage&>(other).data_);
   }
 
-  bool MergePartialFromCodedStream(io::CodedInputStream* input) override;
-
 #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
   const char* _InternalParse(const char* ptr, ParseContext* ctx) final;
+#else
+  bool MergePartialFromCodedStream(io::CodedInputStream* input) override;
 #endif
 
   size_t ByteSizeLong() const override { return data_.size(); }

src/google/protobuf/io/coded_stream.h

@@ -710,6 +710,9 @@ class PROTOBUF_EXPORT CodedOutputStream {
   // Skips a number of bytes, leaving the bytes unmodified in the underlying
   // buffer.  Returns false if an underlying write error occurs.  This is
   // mainly useful with GetDirectBufferPointer().
+  // Note of caution, the skipped bytes may contain uninitialized data. The
+  // caller must make sure that the skipped bytes are properly initialized,
+  // otherwise you might leak bytes from your heap.
   bool Skip(int count);
 
   // Sets *data to point directly at the unwritten part of the

src/google/protobuf/lite_unittest.cc

@@ -44,6 +44,7 @@
 #include <google/protobuf/io/zero_copy_stream_impl_lite.h>
 #include <google/protobuf/wire_format_lite.h>
 #include <gtest/gtest.h>
+#include <google/protobuf/stubs/strutil.h>
 
 namespace google {
 namespace protobuf {
@@ -1049,5 +1050,54 @@ TEST(Lite, MapCrash) {
       "\202\1\15\10\1\200\200\200\200\200\200\200\200\200\200\1"));
 }
 
+TEST(Lite, CorrectEnding) {
+  protobuf_unittest::TestAllTypesLite msg;
+  {
+    // All proto wireformat parsers should act the same on parsing data in as
+    // much as it concerns the parsing, ie. not the interpretation of the data.
+    // TestAllTypesLite is not a group inside another message. So in practice
+    // will not encounter an end-group tag. However the parser should behave
+    // like any wire format parser should.
+    static const char kWireFormat[] = "\204\1";
+    io::CodedInputStream cis(reinterpret_cast<const uint8*>(kWireFormat), 2);
+    // The old CodedInputStream parser got an optimization (ReadTagNoLastTag)
+    // for non-group messages (like TestAllTypesLite) which made it not accept
+    // end-group. This is not a real big deal, but I think going forward its
+    // good to have all parse loops behave 'exactly' the same.
+#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
+    EXPECT_TRUE(msg.MergePartialFromCodedStream(&cis));
+    EXPECT_FALSE(cis.ConsumedEntireMessage());
+    EXPECT_TRUE(cis.LastTagWas(132));
+#endif
+  }
+  {
+    // This is an incomplete end-group tag. This should be a genuine parse
+    // failure.
+    static const char kWireFormat[] = "\214";
+    io::CodedInputStream cis(reinterpret_cast<const uint8*>(kWireFormat), 1);
+    // Unfortunately the old parser detects a parse error in ReadTag and returns
+    // 0 (as it states 0 is an invalid tag). However 0 is not an invalid tag
+    // as it can be used to terminate the stream, so this returns true.
+#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
+    EXPECT_FALSE(msg.MergePartialFromCodedStream(&cis));
+#endif
+  }
+}
+
+TEST(Lite, DebugString) {
+  protobuf_unittest::TestAllTypesLite message1, message2;
+  EXPECT_TRUE(HasPrefixString(message1.DebugString(), "MessageLite at 0x"));
+  EXPECT_TRUE(HasPrefixString(message2.DebugString(), "MessageLite at 0x"));
+
+  // DebugString() and ShortDebugString() are the same for now.
+  EXPECT_EQ(message1.DebugString(), message1.ShortDebugString());
+
+  // Even identical lite protos should have different DebugString() output. Part
+  // of the reason for including the memory address is so that we get some
+  // non-determinism, which should make it easier for us to change the output
+  // later without breaking any code.
+  EXPECT_NE(message1.DebugString(), message2.DebugString());
+}
+
 }  // namespace protobuf
 }  // namespace google

src/google/protobuf/map_entry_lite.h

@@ -214,7 +214,7 @@ class MapEntryImpl : public Base {
           ctx->SetLastTag(tag);
           return ptr;
         }
-        ptr = UnknownFieldParse(tag, nullptr, ptr, ctx);
+        ptr = UnknownFieldParse(tag, static_cast<string*>(nullptr), ptr, ctx);
       }
       GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
     }

src/google/protobuf/map_field.cc

@@ -33,6 +33,8 @@
 
 #include <vector>
 
+#include <google/protobuf/port_def.inc>
+
 namespace google {
 namespace protobuf {
 namespace internal {

src/google/protobuf/message_lite.cc

@@ -34,6 +34,7 @@
 //  Sanjay Ghemawat, Jeff Dean, and others.
 
 #include <climits>
+#include <cstdint>
 #include <string>
 
 #include <google/protobuf/stubs/logging.h>
@@ -48,6 +49,7 @@
 #include <google/protobuf/generated_message_util.h>
 #include <google/protobuf/message_lite.h>
 #include <google/protobuf/repeated_field.h>
+
 #include <google/protobuf/stubs/strutil.h>
 #include <google/protobuf/stubs/stl_util.h>
 
@@ -60,6 +62,11 @@ std::string MessageLite::InitializationErrorString() const {
   return "(cannot determine missing fields for lite message)";
 }
 
+std::string MessageLite::DebugString() const {
+  std::uintptr_t address = reinterpret_cast<std::uintptr_t>(this);
+  return StrCat("MessageLite at 0x", strings::Hex(address));
+}
+
 namespace {
 
 // When serializing, we first compute the byte size, then serialize the message.
@@ -123,7 +130,9 @@ bool MergePartialFromImpl(StringPiece input, MessageLite* msg) {
   const char* ptr;
   internal::ParseContext ctx(io::CodedInputStream::GetDefaultRecursionLimit(),
                              aliasing, &ptr, input);
-  return ctx.AtLegitimateEnd(msg->_InternalParse(ptr, &ctx));
+  ptr = msg->_InternalParse(ptr, &ctx);
+  // ctx has an explicit limit set (length of string_view).
+  return ptr && ctx.EndedAtLimit();
 }
 
 template <bool aliasing>
@@ -131,7 +140,9 @@ bool MergePartialFromImpl(io::ZeroCopyInputStream* input, MessageLite* msg) {
   const char* ptr;
   internal::ParseContext ctx(io::CodedInputStream::GetDefaultRecursionLimit(),
                              aliasing, &ptr, input);
-  return ctx.AtLegitimateEnd(msg->_InternalParse(ptr, &ctx));
+  ptr = msg->_InternalParse(ptr, &ctx);
+  // ctx has no explicit limit (hence we end on end of stream)
+  return ptr && ctx.EndedAtEndOfStream();
 }
 
 template <bool aliasing>
@@ -229,13 +240,15 @@ bool MessageLite::MergePartialFromCodedStream(io::CodedInputStream* input) {
   ctx.data().pool = input->GetExtensionPool();
   ctx.data().factory = input->GetExtensionFactory();
   ptr = _InternalParse(ptr, &ctx);
-  if (!ptr) return false;
+  if (PROTOBUF_PREDICT_FALSE(!ptr)) return false;
   ctx.BackUp(ptr);
-  if (ctx.LastTagMinus1() != 0) {
+  if (!ctx.EndedAtEndOfStream()) {
-    input->SetLastTag(ctx.LastTagMinus1() + 1);
+    GOOGLE_DCHECK(ctx.LastTag() != 1);  // We can't end on a pushed limit.
+    if (ctx.IsExceedingLimit(ptr)) return false;
+    input->SetLastTag(ctx.LastTag());
     return true;
   }
-  if (ctx.AtLimit(ptr)) input->SetConsumed();
+  input->SetConsumed();
   return true;
 }
 #endif  // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER

src/google/protobuf/message_lite.h

@@ -234,6 +234,20 @@ class PROTOBUF_EXPORT MessageLite {
   // results are undefined (probably crash).
   virtual void CheckTypeAndMergeFrom(const MessageLite& other) = 0;
 
+  // These methods return a human-readable summary of the message. Note that
+  // since the MessageLite interface does not support reflection, there is very
+  // little information that these methods can provide. They are shadowed by
+  // methods of the same name on the Message interface which provide much more
+  // information. The methods here are intended primarily to facilitate code
+  // reuse for logic that needs to interoperate with both full and lite protos.
+  //
+  // The format of the returned string is subject to change, so please do not
+  // assume it will remain stable over time.
+  std::string DebugString() const;
+  std::string ShortDebugString() const {
+    return DebugString();
+  }
+
   // Parsing ---------------------------------------------------------
   // Methods for parsing in protocol buffer format.  Most of these are
   // just simple wrappers around MergeFromCodedStream().  Clear() will be

src/google/protobuf/parse_context.cc

@@ -157,11 +157,15 @@ std::pair<const char*, bool> EpsCopyInputStream::DoneFallback(const char* ptr,
   GOOGLE_DCHECK(ptr >= limit_end_);
   int overrun = ptr - buffer_end_;
   GOOGLE_DCHECK(overrun <= kSlopBytes);  // Guaranteed by parse loop.
-  GOOGLE_DCHECK(overrun != limit_);
+  // Did we exceeded the limit (parse error).
-  // We either exceeded the limit (parse error) or we need to get new data.
-  // Did we exceed the limit? Is so parse error.
   if (PROTOBUF_PREDICT_FALSE(overrun > limit_)) return {nullptr, true};
-  GOOGLE_DCHECK(overrun <= limit_);  // Follows from above
+  GOOGLE_DCHECK(overrun != limit_);  // Guaranteed by caller.
+  GOOGLE_DCHECK(overrun < limit_);   // Follows from above
+  // TODO(gerbens) Instead of this dcheck we could just assign, and remove
+  // updating the limit_end from PopLimit, ie.
+  // limit_end_ = buffer_end_ + (std::min)(0, limit_);
+  // if (ptr < limit_end_) return {ptr, false};
+  GOOGLE_DCHECK(limit_end_ == buffer_end_ + (std::min)(0, limit_));
   // At this point we know the following assertion holds.
   GOOGLE_DCHECK(limit_ > 0);
   GOOGLE_DCHECK(limit_end_ == buffer_end_);  // because limit_ > 0
@@ -174,6 +178,8 @@ std::pair<const char*, bool> EpsCopyInputStream::DoneFallback(const char* ptr,
       if (PROTOBUF_PREDICT_FALSE(overrun != 0)) return {nullptr, true};
       GOOGLE_DCHECK(limit_ > 0);
       limit_end_ = buffer_end_;
+      // Distinquish ending on a pushed limit or ending on end-of-stream.
+      SetEndOfStream();
       return {ptr, true};
     }
     limit_ -= buffer_end_ - p;  // Adjust limit_ relative to new anchor
@@ -193,12 +199,19 @@ const char* EpsCopyInputStream::ReadStringFallback(const char* ptr, int size,
   s->clear();
   // TODO(gerbens) assess security. At the moment its parity with
   // CodedInputStream but it allows a payload to reserve large memory.
-  if (size <= buffer_end_ - ptr + limit_) s->reserve(size);
+  if (PROTOBUF_PREDICT_TRUE(size <= buffer_end_ - ptr + limit_)) {
+    s->reserve(size);
+  }
   return AppendStringFallback(ptr, size, s);
 }
 
 const char* EpsCopyInputStream::AppendStringFallback(const char* ptr, int size,
                                                      std::string* str) {
+  // TODO(gerbens) assess security. At the moment its parity with
+  // CodedInputStream but it allows a payload to reserve large memory.
+  if (PROTOBUF_PREDICT_TRUE(size <= buffer_end_ - ptr + limit_)) {
+    str->reserve(size);
+  }
   return AppendSize(ptr, size,
                     [str](const char* p, int s) { str->append(p, s); });
 }
@@ -264,7 +277,6 @@ const char* EpsCopyInputStream::InitFrom(io::ZeroCopyInputStream* zcis) {
   }
   next_chunk_ = nullptr;
   size_ = 0;
-  limit_ = 0;
   limit_end_ = buffer_end_ = buffer_;
   return buffer_;
 }
@@ -340,6 +352,13 @@ bool VerifyUTF8(StringPiece str, const char* field_name) {
   return true;
 }
 
+const char* InlineGreedyStringParser(std::string* s, const char* ptr,
+                                     ParseContext* ctx) {
+  int size = ReadSize(&ptr);
+  if (!ptr) return nullptr;
+  return ctx->ReadString(ptr, size, s);
+}
+
 const char* InlineGreedyStringParserUTF8(std::string* s, const char* ptr,
                                          ParseContext* ctx,
                                          const char* field_name) {
@@ -348,16 +367,6 @@ const char* InlineGreedyStringParserUTF8(std::string* s, const char* ptr,
   return p;
 }
 
-const char* InlineGreedyStringParserUTF8Verify(std::string* s, const char* ptr,
-                                               ParseContext* ctx,
-                                               const char* field_name) {
-  auto p = InlineGreedyStringParser(s, ptr, ctx);
-#ifndef NDEBUG
-  VerifyUTF8(*s, field_name);
-#endif  // !NDEBUG
-  return p;
-}
-
 
 template <typename T, bool sign>
 const char* VarintParser(void* object, const char* ptr, ParseContext* ctx) {
@@ -531,6 +540,12 @@ const char* UnknownFieldParse(uint32 tag, std::string* unknown, const char* ptr,
   return FieldParser(tag, field_parser, ptr, ctx);
 }
 
+const char* UnknownFieldParse(uint32 tag,
+                              InternalMetadataWithArenaLite* metadata,
+                              const char* ptr, ParseContext* ctx) {
+  return UnknownFieldParse(tag, metadata->mutable_unknown_fields(), ptr, ctx);
+}
+
 }  // namespace internal
 }  // namespace protobuf
 }  // namespace google

src/google/protobuf/parse_context.h

@@ -38,6 +38,7 @@
 #include <google/protobuf/io/zero_copy_stream.h>
 #include <google/protobuf/arenastring.h>
 #include <google/protobuf/implicit_weak_message.h>
+#include <google/protobuf/metadata_lite.h>
 #include <google/protobuf/port.h>
 #include <google/protobuf/repeated_field.h>
 #include <google/protobuf/wire_format_lite.h>
@@ -124,17 +125,17 @@ class PROTOBUF_EXPORT EpsCopyInputStream {
   PROTOBUF_MUST_USE_RESULT int PushLimit(const char* ptr, int limit) {
     GOOGLE_DCHECK(limit >= 0);
     limit += ptr - buffer_end_;
-    if (limit < 0) limit_end_ = buffer_end_ + limit;
+    limit_end_ = buffer_end_ + (std::min)(0, limit);
     auto old_limit = limit_;
     limit_ = limit;
     return old_limit - limit;
   }
 
-  PROTOBUF_MUST_USE_RESULT bool PopLimit(int delta, const char* ptr) {
+  PROTOBUF_MUST_USE_RESULT bool PopLimit(int delta) {
-    // Ensure not to forget to check PushLimit return value
+    if (PROTOBUF_PREDICT_FALSE(!EndedAtLimit())) return false;
-    GOOGLE_DCHECK(delta >= 0);
-    if (ptr == nullptr || ptr - buffer_end_ != limit_) return false;
     limit_ = limit_ + delta;
+    // TODO(gerbens) We could remove this line and hoist the code to
+    // DoneFallback. Study the perf/bin-size effects.
     limit_end_ = buffer_end_ + (std::min)(0, limit_);
     return true;
   }
@@ -175,9 +176,19 @@ class PROTOBUF_EXPORT EpsCopyInputStream {
   PROTOBUF_MUST_USE_RESULT const char* ReadPackedVarint(const char* ptr,
                                                         Add add);
 
-  bool AtLimit(const char* ptr) const {
+  uint32 LastTag() const { return last_tag_minus_1_ + 1; }
-    return (ptr - buffer_end_ == limit_) ||
+  bool ConsumeEndGroup(uint32 start_tag) {
-           (next_chunk_ == nullptr && limit_ > 0 && ptr == buffer_end_);
+    bool res = last_tag_minus_1_ == start_tag;
+    last_tag_minus_1_ = 0;
+    return res;
+  }
+  bool EndedAtLimit() const { return last_tag_minus_1_ == 0; }
+  bool EndedAtEndOfStream() const { return last_tag_minus_1_ == 1; }
+  void SetLastTag(uint32 tag) { last_tag_minus_1_ = tag - 1; }
+  void SetEndOfStream() { last_tag_minus_1_ = 1; }
+  bool IsExceedingLimit(const char* ptr) {
+    return ptr > limit_end_ &&
+           (next_chunk_ == nullptr || ptr - buffer_end_ > limit_);
   }
 
  protected:
@@ -233,6 +244,20 @@ class PROTOBUF_EXPORT EpsCopyInputStream {
   char buffer_[2 * kSlopBytes] = {};
   enum { kNoAliasing = 0, kOnPatch = 1, kNoDelta = 2 };
   std::uintptr_t aliasing_ = kNoAliasing;
+  // This variable is used to communicate how the parse ended, in order to
+  // completely verify the parsed data. A wire-format parse can end because of
+  // one of the following conditions:
+  // 1) A parse can end on a pushed limit.
+  // 2) A parse can end on End Of Stream (EOS).
+  // 3) A parse can end on 0 tag (only valid for toplevel message).
+  // 4) A parse can end on an end-group tag.
+  // This variable should always be set to 0, which indicates case 1. If the
+  // parse terminated due to EOS (case 2), it's set to 1. In case the parse
+  // ended due to a terminating tag (case 3 and 4) it's set to (tag - 1).
+  // This var doesn't really belong in EpsCopyInputStream and should be part of
+  // the ParseContext, but case 2 is most easily and optimally implemented in
+  // DoneFallback.
+  uint32 last_tag_minus_1_ = 0;
 
   std::pair<const char*, bool> DoneFallback(const char* ptr, int d);
   const char* Next(int overrun, int d);
@@ -306,12 +331,6 @@ class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream {
   bool DoneNoSlopCheck(const char** ptr) { return DoneWithCheck(ptr, -1); }
 
   int depth() const { return depth_; }
-  void SetLastTag(uint32 tag) { last_tag_minus_1_ = tag - 1; }
-  uint32 LastTagMinus1() const { return last_tag_minus_1_; }
-
-  bool AtLegitimateEnd(const char* ptr) const {
-    return ptr && AtLimit(ptr) && last_tag_minus_1_ == 0;
-  }
 
   Data& data() { return data_; }
   const Data& data() const { return data_; }
@@ -331,8 +350,7 @@ class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream {
     ptr = msg->_InternalParse(ptr, this);
     group_depth_--;
     depth_++;
-    if (last_tag_minus_1_ != tag) return nullptr;
+    if (PROTOBUF_PREDICT_FALSE(!ConsumeEndGroup(tag))) return nullptr;
-    last_tag_minus_1_ = 0;
     return ptr;
   }
 
@@ -346,7 +364,6 @@ class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream {
   // Unfortunately necessary for the fringe case of ending on 0 or end-group tag
   // in the last kSlopBytes of a ZeroCopyInputStream chunk.
   int group_depth_ = INT_MIN;
-  uint32 last_tag_minus_1_ = 0;
   Data data_;
 };
 
@@ -403,12 +420,26 @@ inline uint32 DecodeTwoBytes(uint32 value, const char** ptr) {
 
 // Used for tags, could read up to 5 bytes which must be available.
 // Caller must ensure its safe to call.
-inline const char* ReadTag(const char* p, uint32* out) {
-  return VarintParse<5>(p, out);
-}
 
 std::pair<const char*, uint32> ReadTagFallback(const char* p, uint32 res);
 
+inline const char* ReadTag(const char* p, uint32* out) {
+  uint32 res = static_cast<uint8>(p[0]);
+  if (res < 128) {
+    *out = res;
+    return p + 1;
+  }
+  uint32 second = static_cast<uint8>(p[1]);
+  res += (second - 1) << 7;
+  if (second < 128) {
+    *out = res;
+    return p + 2;
+  }
+  auto tmp = ReadTagFallback(p + 2, res);
+  *out = tmp.second;
+  return tmp.first;
+}
+
 // Will preload the next 2 bytes
 inline const char* ReadTag(const char* p, uint32* out, uint32* preload) {
   uint32 res = static_cast<uint8>(p[0]);
@@ -536,10 +567,11 @@ PROTOBUF_MUST_USE_RESULT const char* ParseContext::ParseMessage(
   int size = ReadSize(&ptr);
   if (!ptr) return nullptr;
   auto old = PushLimit(ptr, size);
-  if (--depth_ < 0 || old < 0) return nullptr;
+  if (--depth_ < 0) return nullptr;
   ptr = msg->_InternalParse(ptr, this);
+  if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) return nullptr;
   depth_++;
-  if (!PopLimit(old, ptr) || last_tag_minus_1_ != 0) return nullptr;
+  if (!PopLimit(old)) return nullptr;
   return ptr;
 }
 
@@ -555,7 +587,7 @@ const char* EpsCopyInputStream::ReadPackedVarint(const char* ptr, Add add) {
     if (!ptr) return nullptr;
     add(varint);
   }
-  if (!PopLimit(old, ptr)) return nullptr;
+  if (!PopLimit(old)) return nullptr;
   return ptr;
 }
 
@@ -564,19 +596,22 @@ PROTOBUF_EXPORT
 bool VerifyUTF8(StringPiece s, const char* field_name);
 
 // All the string parsers with or without UTF checking and for all CTypes.
-inline PROTOBUF_MUST_USE_RESULT const char* InlineGreedyStringParser(
+PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* InlineGreedyStringParser(
-    std::string* s, const char* ptr, ParseContext* ctx) {
+    std::string* s, const char* ptr, ParseContext* ctx);
-  int size = ReadSize(&ptr);
-  if (!ptr) return nullptr;
-  return ctx->ReadString(ptr, size, s);
-}
 
 PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char*
 InlineGreedyStringParserUTF8(std::string* s, const char* ptr, ParseContext* ctx,
                              const char* field_name);
-PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char*
+// Inline because we don't want to pay the price of field_name in opt mode.
-InlineGreedyStringParserUTF8Verify(std::string* s, const char* ptr,
+inline PROTOBUF_MUST_USE_RESULT const char* InlineGreedyStringParserUTF8Verify(
-                                   ParseContext* ctx, const char* field_name);
+    std::string* s, const char* ptr, ParseContext* ctx,
+    const char* field_name) {
+  auto p = InlineGreedyStringParser(s, ptr, ctx);
+#ifndef NDEBUG
+  VerifyUTF8(*s, field_name);
+#endif  // !NDEBUG
+  return p;
+}
 
 
 // Add any of the following lines to debug which parse function is failing.
@@ -705,6 +740,9 @@ PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* UnknownGroupLiteParse(
 // UnknownFieldSet* to make the generated code isomorphic between full and lite.
 PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* UnknownFieldParse(
     uint32 tag, std::string* unknown, const char* ptr, ParseContext* ctx);
+PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* UnknownFieldParse(
+    uint32 tag, InternalMetadataWithArenaLite* metadata, const char* ptr,
+    ParseContext* ctx);
 
 }  // namespace internal
 }  // namespace protobuf

src/google/protobuf/reflection_ops.cc

@@ -46,6 +46,8 @@
 #include <google/protobuf/stubs/strutil.h>
 
 
+#include <google/protobuf/port_def.inc>
+
 namespace google {
 namespace protobuf {
 namespace internal {

src/google/protobuf/repeated_field.h

@@ -2282,14 +2282,23 @@ namespace internal {
 // This code based on net/proto/proto-array-internal.h by Jeffrey Yasskin
 // (jyasskin@google.com).
 template<typename Element>
-class RepeatedPtrIterator {
+class RepeatedPtrIterator
+    : public std::iterator<
+          std::random_access_iterator_tag, Element> {
  public:
   typedef RepeatedPtrIterator<Element> iterator;
-  typedef std::random_access_iterator_tag iterator_category;
+  typedef std::iterator<
+          std::random_access_iterator_tag, Element> superclass;
+
+  // Shadow the value_type in std::iterator<> because const_iterator::value_type
+  // needs to be T, not const T.
   typedef typename std::remove_const<Element>::type value_type;
-  typedef std::ptrdiff_t difference_type;
+
-  typedef Element* pointer;
+  // Let the compiler know that these are type names, so we don't have to
-  typedef Element& reference;
+  // write "typename" in front of them everywhere.
+  typedef typename superclass::reference reference;
+  typedef typename superclass::pointer pointer;
+  typedef typename superclass::difference_type difference_type;
 
   RepeatedPtrIterator() : it_(NULL) {}
   explicit RepeatedPtrIterator(void* const* it) : it_(it) {}
@@ -2369,14 +2378,21 @@ class RepeatedPtrIterator {
 // referenced by the iterator.  It should either be "void *" for a mutable
 // iterator, or "const void* const" for a constant iterator.
 template <typename Element, typename VoidPtr>
-class RepeatedPtrOverPtrsIterator {
+class RepeatedPtrOverPtrsIterator
+    : public std::iterator<std::random_access_iterator_tag, Element> {
  public:
   typedef RepeatedPtrOverPtrsIterator<Element, VoidPtr> iterator;
-  typedef std::random_access_iterator_tag iterator_category;
+  typedef std::iterator<std::random_access_iterator_tag, Element> superclass;
+
+  // Shadow the value_type in std::iterator<> because const_iterator::value_type
+  // needs to be T, not const T.
   typedef typename std::remove_const<Element>::type value_type;
-  typedef std::ptrdiff_t difference_type;
+
-  typedef Element* pointer;
+  // Let the compiler know that these are type names, so we don't have to
-  typedef Element& reference;
+  // write "typename" in front of them everywhere.
+  typedef typename superclass::reference reference;
+  typedef typename superclass::pointer pointer;
+  typedef typename superclass::difference_type difference_type;
 
   RepeatedPtrOverPtrsIterator() : it_(NULL) {}
   explicit RepeatedPtrOverPtrsIterator(VoidPtr* it) : it_(it) {}

src/google/protobuf/test_messages_proto2.proto

@@ -33,6 +33,8 @@
 // - conformance tests
 //
 
+// LINT: ALLOW_GROUPS
+
 syntax = "proto2";
 
 package protobuf_test_messages.proto2;
@@ -180,6 +182,9 @@ message TestAllTypesProto2 {
   optional int32 field_name17__ = 417;
   optional int32 Field_name18__ = 418;
 
+  // Reserved for unknown fields test.
+  reserved 1000 to 9999;
+
   // message_set test case.
   message MessageSetCorrect {
     option message_set_wire_format = true;
@@ -214,3 +219,15 @@ enum ForeignEnumProto2 {
 extend TestAllTypesProto2 {
   optional int32 extension_int32 = 120;
 }
+
+message UnknownToTestAllTypes {
+  optional int32  optional_int32 = 1001;
+  optional string optional_string = 1002;
+  optional ForeignMessageProto2 nested_message = 1003;
+  optional group OptionalGroup = 1004 {
+    optional int32 a = 1;
+  }
+  optional bool optional_bool = 1006;
+  repeated int32 repeated_int32 = 1011;
+}
+

src/google/protobuf/text_format.cc

@@ -64,6 +64,8 @@
 #include <google/protobuf/stubs/map_util.h>
 #include <google/protobuf/stubs/stl_util.h>
 
+#include <google/protobuf/port_def.inc>
+
 
 namespace google {
 namespace protobuf {

src/google/protobuf/unknown_field_set.cc

@@ -357,6 +357,11 @@ const char* UnknownFieldParse(uint64 tag, UnknownFieldSet* unknown,
   return FieldParser(tag, field_parser, ptr, ctx);
 }
 
+const char* UnknownFieldParse(uint32 tag, InternalMetadataWithArena* metadata,
+                              const char* ptr, ParseContext* ctx) {
+  return UnknownFieldParse(tag, metadata->mutable_unknown_fields(), ptr, ctx);
+}
+
 }  // namespace internal
 #endif  // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
 

src/google/protobuf/unknown_field_set.h

@@ -212,6 +212,9 @@ const char* UnknownGroupParse(UnknownFieldSet* unknown, const char* ptr,
 PROTOBUF_EXPORT
 const char* UnknownFieldParse(uint64 tag, UnknownFieldSet* unknown,
                               const char* ptr, ParseContext* ctx);
+PROTOBUF_EXPORT
+const char* UnknownFieldParse(uint32 tag, InternalMetadataWithArena* metadata,
+                              const char* ptr, ParseContext* ctx);
 
 }  // namespace internal
 #endif  // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER

src/google/protobuf/util/json_format.proto

@@ -35,8 +35,8 @@
 // A proto file we will use for unit testing.
 
 syntax = "proto2";
-package protobuf_unittest;
 
+package protobuf_unittest;
 
 message TestFlagsAndStrings {
   required int32 A = 1;

src/google/protobuf/util/json_format_proto3.proto

@@ -32,17 +32,17 @@ syntax = "proto3";
 
 package proto3;
 
-option java_package = "com.google.protobuf.util";
+import "google/protobuf/any.proto";
-option java_outer_classname = "JsonFormatProto3";
-
 import "google/protobuf/duration.proto";
+import "google/protobuf/field_mask.proto";
+import "google/protobuf/struct.proto";
 import "google/protobuf/timestamp.proto";
 import "google/protobuf/wrappers.proto";
-import "google/protobuf/struct.proto";
-import "google/protobuf/any.proto";
-import "google/protobuf/field_mask.proto";
 import "google/protobuf/unittest.proto";
 
+option java_package = "com.google.protobuf.util";
+option java_outer_classname = "JsonFormatProto3";
+
 enum EnumType {
   FOO = 0;
   BAR = 1;

src/google/protobuf/util/message_differencer.h

@@ -127,7 +127,7 @@ class PROTOBUF_EXPORT MessageDifferencer {
   // defined as all fields within the two messages having the same value. This
   // differs from the Equals method above in that fields with default values
   // are considered set to said value automatically. For details on how default
-  // values are defined for each field type, see 
+  // values are defined for each field type, see:
   // https://developers.google.com/protocol-buffers/docs/proto?csw=1#optional.
   // Also, Equivalent() ignores unknown fields. Use IgnoreField() and Compare()
   // if some fields should be ignored in the comparison.

src/google/protobuf/util/message_differencer_unittest.proto

@@ -35,6 +35,7 @@
 // This file contains messages for testing repeated field comparison
 
 syntax = "proto2";
+
 package protobuf_unittest;
 
 option optimize_for = SPEED;
@@ -53,22 +54,21 @@ message TestField {
 
 message TestDiffMessage {
   repeated group Item = 1 {
-    optional int32  a  =  2;     // Test basic repeated field comparison.
+    optional int32 a = 2;       // Test basic repeated field comparison.
-    optional string b  =  4;     // Test basic repeated field comparison.
+    optional string b = 4;      // Test basic repeated field comparison.
-    repeated int32  ra =  3;     // Test SetOfSet Comparison.
+    repeated int32 ra = 3;      // Test SetOfSet Comparison.
-    repeated string rb =  5;     // Test TreatAsMap when key is repeated
+    repeated string rb = 5;     // Test TreatAsMap when key is repeated
-    optional TestField m  = 6;   // Test TreatAsMap when key is a message
+    optional TestField m = 6;   // Test TreatAsMap when key is a message
-    repeated TestField rm = 7;   // Test TreatAsMap when key is a repeated
+    repeated TestField rm = 7;  // Test TreatAsMap when key is a repeated
-                                 // message
+                                // message
   }
 
-  optional int32  v  = 13 [deprecated = true];
+  optional int32 v = 13 [deprecated = true];
-  optional string w  = 14;
+  optional string w = 14;
-  optional TestField m  = 15;
+  optional TestField m = 15;
-  repeated int32  rv = 11;       // Test for combinations
+  repeated int32 rv = 11;                          // Test for combinations
-  repeated string rw = 10;       // Test for combinations
+  repeated string rw = 10;                         // Test for combinations
-  repeated TestField rm = 12 [deprecated = true];    // Test for combinations
+  repeated TestField rm = 12 [deprecated = true];  // Test for combinations
 
   extensions 100 to 199;
 }
-

src/google/protobuf/wire_format.cc

@@ -52,6 +52,8 @@
 #include <google/protobuf/unknown_field_set.h>
 
 
+#include <google/protobuf/port_def.inc>
+
 const size_t kMapEntryTagByteSize = 2;
 
 namespace google {

src/google/protobuf/wire_format_lite.h

@@ -248,16 +248,6 @@ class PROTOBUF_EXPORT WireFormatLite {
   // of these methods are defined in wire_format_lite_inl.h; you must #include
   // that file to use these.
 
-#ifdef NDEBUG
-#define INL PROTOBUF_ALWAYS_INLINE
-#else
-// Avoid excessive inlining in non-optimized builds. Without other optimizations
-// the inlining is not going to provide benefits anyway and the huge resulting
-// functions, especially in the proto-generated serialization functions, produce
-// stack frames so large that many tests run into stack overflows (b/32192897).
-#define INL
-#endif
-
   // Read fields, not including tags.  The assumption is that you already
   // read the tag to determine what field to read.
 
@@ -265,15 +255,16 @@ class PROTOBUF_EXPORT WireFormatLite {
   // the represented type and the FieldType. These are specialized with the
   // appropriate definition for each declared type.
   template <typename CType, enum FieldType DeclaredType>
-  INL static bool ReadPrimitive(io::CodedInputStream* input, CType* value);
+  PROTOBUF_ALWAYS_INLINE static bool ReadPrimitive(io::CodedInputStream* input,
+                                                   CType* value);
 
   // Reads repeated primitive values, with optimizations for repeats.
   // tag_size and tag should both be compile-time constants provided by the
   // protocol compiler.
   template <typename CType, enum FieldType DeclaredType>
-  INL static bool ReadRepeatedPrimitive(int tag_size, uint32 tag,
+  PROTOBUF_ALWAYS_INLINE static bool ReadRepeatedPrimitive(
-                                        io::CodedInputStream* input,
+      int tag_size, uint32 tag, io::CodedInputStream* input,
-                                        RepeatedField<CType>* value);
+      RepeatedField<CType>* value);
 
   // Identical to ReadRepeatedPrimitive, except will not inline the
   // implementation.
@@ -288,15 +279,15 @@ class PROTOBUF_EXPORT WireFormatLite {
   // This is only implemented for the types with fixed wire size, e.g.
   // float, double, and the (s)fixed* types.
   template <typename CType, enum FieldType DeclaredType>
-  INL static const uint8* ReadPrimitiveFromArray(const uint8* buffer,
+  PROTOBUF_ALWAYS_INLINE static const uint8* ReadPrimitiveFromArray(
-                                                 CType* value);
+      const uint8* buffer, CType* value);
 
   // Reads a primitive packed field.
   //
   // This is only implemented for packable types.
   template <typename CType, enum FieldType DeclaredType>
-  INL static bool ReadPackedPrimitive(io::CodedInputStream* input,
+  PROTOBUF_ALWAYS_INLINE static bool ReadPackedPrimitive(
-                                      RepeatedField<CType>* value);
+      io::CodedInputStream* input, RepeatedField<CType>* value);
 
   // Identical to ReadPackedPrimitive, except will not inline the
   // implementation.
@@ -364,28 +355,38 @@ class PROTOBUF_EXPORT WireFormatLite {
   // Write a tag.  The Write*() functions typically include the tag, so
   // normally there's no need to call this unless using the Write*NoTag()
   // variants.
-  INL static void WriteTag(int field_number, WireType type,
+  PROTOBUF_ALWAYS_INLINE static void WriteTag(int field_number, WireType type,
-                           io::CodedOutputStream* output);
+                                              io::CodedOutputStream* output);
 
   // Write fields, without tags.
-  INL static void WriteInt32NoTag(int32 value, io::CodedOutputStream* output);
+  PROTOBUF_ALWAYS_INLINE static void WriteInt32NoTag(
-  INL static void WriteInt64NoTag(int64 value, io::CodedOutputStream* output);
+      int32 value, io::CodedOutputStream* output);
-  INL static void WriteUInt32NoTag(uint32 value, io::CodedOutputStream* output);
+  PROTOBUF_ALWAYS_INLINE static void WriteInt64NoTag(
-  INL static void WriteUInt64NoTag(uint64 value, io::CodedOutputStream* output);
+      int64 value, io::CodedOutputStream* output);
-  INL static void WriteSInt32NoTag(int32 value, io::CodedOutputStream* output);
+  PROTOBUF_ALWAYS_INLINE static void WriteUInt32NoTag(
-  INL static void WriteSInt64NoTag(int64 value, io::CodedOutputStream* output);
+      uint32 value, io::CodedOutputStream* output);
-  INL static void WriteFixed32NoTag(uint32 value,
+  PROTOBUF_ALWAYS_INLINE static void WriteUInt64NoTag(
-                                    io::CodedOutputStream* output);
+      uint64 value, io::CodedOutputStream* output);
-  INL static void WriteFixed64NoTag(uint64 value,
+  PROTOBUF_ALWAYS_INLINE static void WriteSInt32NoTag(
-                                    io::CodedOutputStream* output);
+      int32 value, io::CodedOutputStream* output);
-  INL static void WriteSFixed32NoTag(int32 value,
+  PROTOBUF_ALWAYS_INLINE static void WriteSInt64NoTag(
-                                     io::CodedOutputStream* output);
+      int64 value, io::CodedOutputStream* output);
-  INL static void WriteSFixed64NoTag(int64 value,
+  PROTOBUF_ALWAYS_INLINE static void WriteFixed32NoTag(
-                                     io::CodedOutputStream* output);
+      uint32 value, io::CodedOutputStream* output);
-  INL static void WriteFloatNoTag(float value, io::CodedOutputStream* output);
+  PROTOBUF_ALWAYS_INLINE static void WriteFixed64NoTag(
-  INL static void WriteDoubleNoTag(double value, io::CodedOutputStream* output);
+      uint64 value, io::CodedOutputStream* output);
-  INL static void WriteBoolNoTag(bool value, io::CodedOutputStream* output);
+  PROTOBUF_ALWAYS_INLINE static void WriteSFixed32NoTag(
-  INL static void WriteEnumNoTag(int value, io::CodedOutputStream* output);
+      int32 value, io::CodedOutputStream* output);
+  PROTOBUF_ALWAYS_INLINE static void WriteSFixed64NoTag(
+      int64 value, io::CodedOutputStream* output);
+  PROTOBUF_ALWAYS_INLINE static void WriteFloatNoTag(
+      float value, io::CodedOutputStream* output);
+  PROTOBUF_ALWAYS_INLINE static void WriteDoubleNoTag(
+      double value, io::CodedOutputStream* output);
+  PROTOBUF_ALWAYS_INLINE static void WriteBoolNoTag(
+      bool value, io::CodedOutputStream* output);
+  PROTOBUF_ALWAYS_INLINE static void WriteEnumNoTag(
+      int value, io::CodedOutputStream* output);
 
   // Write array of primitive fields, without tags
   static void WriteFloatArray(const float* a, int n,
@@ -468,147 +469,161 @@ class PROTOBUF_EXPORT WireFormatLite {
                                            io::CodedOutputStream* output);
 
   // Like above, but use only *ToArray methods of CodedOutputStream.
-  INL static uint8* WriteTagToArray(int field_number, WireType type,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteTagToArray(int field_number,
-                                    uint8* target);
+                                                       WireType type,
+                                                       uint8* target);
 
   // Write fields, without tags.
-  INL static uint8* WriteInt32NoTagToArray(int32 value, uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteInt32NoTagToArray(int32 value,
-  INL static uint8* WriteInt64NoTagToArray(int64 value, uint8* target);
+                                                              uint8* target);
-  INL static uint8* WriteUInt32NoTagToArray(uint32 value, uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteInt64NoTagToArray(int64 value,
-  INL static uint8* WriteUInt64NoTagToArray(uint64 value, uint8* target);
+                                                              uint8* target);
-  INL static uint8* WriteSInt32NoTagToArray(int32 value, uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt32NoTagToArray(uint32 value,
-  INL static uint8* WriteSInt64NoTagToArray(int64 value, uint8* target);
+                                                               uint8* target);
-  INL static uint8* WriteFixed32NoTagToArray(uint32 value, uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt64NoTagToArray(uint64 value,
-  INL static uint8* WriteFixed64NoTagToArray(uint64 value, uint8* target);
+                                                               uint8* target);
-  INL static uint8* WriteSFixed32NoTagToArray(int32 value, uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt32NoTagToArray(int32 value,
-  INL static uint8* WriteSFixed64NoTagToArray(int64 value, uint8* target);
+                                                               uint8* target);
-  INL static uint8* WriteFloatNoTagToArray(float value, uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt64NoTagToArray(int64 value,
-  INL static uint8* WriteDoubleNoTagToArray(double value, uint8* target);
+                                                               uint8* target);
-  INL static uint8* WriteBoolNoTagToArray(bool value, uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed32NoTagToArray(uint32 value,
-  INL static uint8* WriteEnumNoTagToArray(int value, uint8* target);
+                                                                uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed64NoTagToArray(uint64 value,
+                                                                uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed32NoTagToArray(int32 value,
+                                                                 uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed64NoTagToArray(int64 value,
+                                                                 uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteFloatNoTagToArray(float value,
+                                                              uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteDoubleNoTagToArray(double value,
+                                                               uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteBoolNoTagToArray(bool value,
+                                                             uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteEnumNoTagToArray(int value,
+                                                             uint8* target);
 
   // Write fields, without tags.  These require that value.size() > 0.
   template <typename T>
-  INL static uint8* WritePrimitiveNoTagToArray(const RepeatedField<T>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WritePrimitiveNoTagToArray(
-                                               uint8* (*Writer)(T, uint8*),
+      const RepeatedField<T>& value, uint8* (*Writer)(T, uint8*),
-                                               uint8* target);
+      uint8* target);
   template <typename T>
-  INL static uint8* WriteFixedNoTagToArray(const RepeatedField<T>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteFixedNoTagToArray(
-                                           uint8* (*Writer)(T, uint8*),
+      const RepeatedField<T>& value, uint8* (*Writer)(T, uint8*),
-                                           uint8* target);
+      uint8* target);
-
+
-  INL static uint8* WriteInt32NoTagToArray(const RepeatedField<int32>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteInt32NoTagToArray(
-                                           uint8* output);
+      const RepeatedField<int32>& value, uint8* output);
-  INL static uint8* WriteInt64NoTagToArray(const RepeatedField<int64>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteInt64NoTagToArray(
-                                           uint8* output);
+      const RepeatedField<int64>& value, uint8* output);
-  INL static uint8* WriteUInt32NoTagToArray(const RepeatedField<uint32>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt32NoTagToArray(
-                                            uint8* output);
+      const RepeatedField<uint32>& value, uint8* output);
-  INL static uint8* WriteUInt64NoTagToArray(const RepeatedField<uint64>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt64NoTagToArray(
-                                            uint8* output);
+      const RepeatedField<uint64>& value, uint8* output);
-  INL static uint8* WriteSInt32NoTagToArray(const RepeatedField<int32>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt32NoTagToArray(
-                                            uint8* output);
+      const RepeatedField<int32>& value, uint8* output);
-  INL static uint8* WriteSInt64NoTagToArray(const RepeatedField<int64>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt64NoTagToArray(
-                                            uint8* output);
+      const RepeatedField<int64>& value, uint8* output);
-  INL static uint8* WriteFixed32NoTagToArray(const RepeatedField<uint32>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed32NoTagToArray(
-                                             uint8* output);
+      const RepeatedField<uint32>& value, uint8* output);
-  INL static uint8* WriteFixed64NoTagToArray(const RepeatedField<uint64>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed64NoTagToArray(
-                                             uint8* output);
+      const RepeatedField<uint64>& value, uint8* output);
-  INL static uint8* WriteSFixed32NoTagToArray(const RepeatedField<int32>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed32NoTagToArray(
-                                              uint8* output);
+      const RepeatedField<int32>& value, uint8* output);
-  INL static uint8* WriteSFixed64NoTagToArray(const RepeatedField<int64>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed64NoTagToArray(
-                                              uint8* output);
+      const RepeatedField<int64>& value, uint8* output);
-  INL static uint8* WriteFloatNoTagToArray(const RepeatedField<float>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteFloatNoTagToArray(
-                                           uint8* output);
+      const RepeatedField<float>& value, uint8* output);
-  INL static uint8* WriteDoubleNoTagToArray(const RepeatedField<double>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteDoubleNoTagToArray(
-                                            uint8* output);
+      const RepeatedField<double>& value, uint8* output);
-  INL static uint8* WriteBoolNoTagToArray(const RepeatedField<bool>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteBoolNoTagToArray(
-                                          uint8* output);
+      const RepeatedField<bool>& value, uint8* output);
-  INL static uint8* WriteEnumNoTagToArray(const RepeatedField<int>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteEnumNoTagToArray(
-                                          uint8* output);
+      const RepeatedField<int>& value, uint8* output);
 
   // Write fields, including tags.
-  INL static uint8* WriteInt32ToArray(int field_number, int32 value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteInt32ToArray(int field_number,
-                                      uint8* target);
+                                                         int32 value,
-  INL static uint8* WriteInt64ToArray(int field_number, int64 value,
+                                                         uint8* target);
-                                      uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteInt64ToArray(int field_number,
-  INL static uint8* WriteUInt32ToArray(int field_number, uint32 value,
+                                                         int64 value,
-                                       uint8* target);
+                                                         uint8* target);
-  INL static uint8* WriteUInt64ToArray(int field_number, uint64 value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt32ToArray(int field_number,
-                                       uint8* target);
+                                                          uint32 value,
-  INL static uint8* WriteSInt32ToArray(int field_number, int32 value,
+                                                          uint8* target);
-                                       uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt64ToArray(int field_number,
-  INL static uint8* WriteSInt64ToArray(int field_number, int64 value,
+                                                          uint64 value,
-                                       uint8* target);
+                                                          uint8* target);
-  INL static uint8* WriteFixed32ToArray(int field_number, uint32 value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt32ToArray(int field_number,
-                                        uint8* target);
+                                                          int32 value,
-  INL static uint8* WriteFixed64ToArray(int field_number, uint64 value,
+                                                          uint8* target);
-                                        uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt64ToArray(int field_number,
-  INL static uint8* WriteSFixed32ToArray(int field_number, int32 value,
+                                                          int64 value,
-                                         uint8* target);
+                                                          uint8* target);
-  INL static uint8* WriteSFixed64ToArray(int field_number, int64 value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed32ToArray(int field_number,
-                                         uint8* target);
+                                                           uint32 value,
-  INL static uint8* WriteFloatToArray(int field_number, float value,
+                                                           uint8* target);
-                                      uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed64ToArray(int field_number,
-  INL static uint8* WriteDoubleToArray(int field_number, double value,
+                                                           uint64 value,
-                                       uint8* target);
+                                                           uint8* target);
-  INL static uint8* WriteBoolToArray(int field_number, bool value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed32ToArray(int field_number,
-                                     uint8* target);
+                                                            int32 value,
-  INL static uint8* WriteEnumToArray(int field_number, int value,
+                                                            uint8* target);
-                                     uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed64ToArray(int field_number,
+                                                            int64 value,
+                                                            uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteFloatToArray(int field_number,
+                                                         float value,
+                                                         uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteDoubleToArray(int field_number,
+                                                          double value,
+                                                          uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteBoolToArray(int field_number,
+                                                        bool value,
+                                                        uint8* target);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteEnumToArray(int field_number,
+                                                        int value,
+                                                        uint8* target);
 
   template <typename T>
-  INL static uint8* WritePrimitiveToArray(int field_number,
+  PROTOBUF_ALWAYS_INLINE static uint8* WritePrimitiveToArray(
-                                          const RepeatedField<T>& value,
+      int field_number, const RepeatedField<T>& value,
-                                          uint8* (*Writer)(int, T, uint8*),
+      uint8* (*Writer)(int, T, uint8*), uint8* target);
-                                          uint8* target);
+
-
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteInt32ToArray(
-  INL static uint8* WriteInt32ToArray(int field_number,
+      int field_number, const RepeatedField<int32>& value, uint8* output);
-                                      const RepeatedField<int32>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteInt64ToArray(
-                                      uint8* output);
+      int field_number, const RepeatedField<int64>& value, uint8* output);
-  INL static uint8* WriteInt64ToArray(int field_number,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt32ToArray(
-                                      const RepeatedField<int64>& value,
+      int field_number, const RepeatedField<uint32>& value, uint8* output);
-                                      uint8* output);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt64ToArray(
-  INL static uint8* WriteUInt32ToArray(int field_number,
+      int field_number, const RepeatedField<uint64>& value, uint8* output);
-                                       const RepeatedField<uint32>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt32ToArray(
-                                       uint8* output);
+      int field_number, const RepeatedField<int32>& value, uint8* output);
-  INL static uint8* WriteUInt64ToArray(int field_number,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt64ToArray(
-                                       const RepeatedField<uint64>& value,
+      int field_number, const RepeatedField<int64>& value, uint8* output);
-                                       uint8* output);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed32ToArray(
-  INL static uint8* WriteSInt32ToArray(int field_number,
+      int field_number, const RepeatedField<uint32>& value, uint8* output);
-                                       const RepeatedField<int32>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed64ToArray(
-                                       uint8* output);
+      int field_number, const RepeatedField<uint64>& value, uint8* output);
-  INL static uint8* WriteSInt64ToArray(int field_number,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed32ToArray(
-                                       const RepeatedField<int64>& value,
+      int field_number, const RepeatedField<int32>& value, uint8* output);
-                                       uint8* output);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed64ToArray(
-  INL static uint8* WriteFixed32ToArray(int field_number,
+      int field_number, const RepeatedField<int64>& value, uint8* output);
-                                        const RepeatedField<uint32>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteFloatToArray(
-                                        uint8* output);
+      int field_number, const RepeatedField<float>& value, uint8* output);
-  INL static uint8* WriteFixed64ToArray(int field_number,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteDoubleToArray(
-                                        const RepeatedField<uint64>& value,
+      int field_number, const RepeatedField<double>& value, uint8* output);
-                                        uint8* output);
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteBoolToArray(
-  INL static uint8* WriteSFixed32ToArray(int field_number,
+      int field_number, const RepeatedField<bool>& value, uint8* output);
-                                         const RepeatedField<int32>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteEnumToArray(
-                                         uint8* output);
+      int field_number, const RepeatedField<int>& value, uint8* output);
-  INL static uint8* WriteSFixed64ToArray(int field_number,
+
-                                         const RepeatedField<int64>& value,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteStringToArray(
-                                         uint8* output);
+      int field_number, const std::string& value, uint8* target);
-  INL static uint8* WriteFloatToArray(int field_number,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteBytesToArray(
-                                      const RepeatedField<float>& value,
+      int field_number, const std::string& value, uint8* target);
-                                      uint8* output);
-  INL static uint8* WriteDoubleToArray(int field_number,
-                                       const RepeatedField<double>& value,
-                                       uint8* output);
-  INL static uint8* WriteBoolToArray(int field_number,
-                                     const RepeatedField<bool>& value,
-                                     uint8* output);
-  INL static uint8* WriteEnumToArray(int field_number,
-                                     const RepeatedField<int>& value,
-                                     uint8* output);
-
-  INL static uint8* WriteStringToArray(int field_number,
-                                       const std::string& value, uint8* target);
-  INL static uint8* WriteBytesToArray(int field_number,
-                                      const std::string& value, uint8* target);
 
   // Whether to serialize deterministically (e.g., map keys are
   // sorted) is a property of a CodedOutputStream, and in the process
@@ -616,39 +631,33 @@ class PROTOBUF_EXPORT WireFormatLite {
   // have a CodedOutputStream available, so they get an additional parameter
   // telling them whether to serialize deterministically.
   template <typename MessageType>
-  INL static uint8* InternalWriteGroupToArray(int field_number,
+  PROTOBUF_ALWAYS_INLINE static uint8* InternalWriteGroupToArray(
-                                              const MessageType& value,
+      int field_number, const MessageType& value, uint8* target);
-                                              uint8* target);
   template <typename MessageType>
-  INL static uint8* InternalWriteMessageToArray(int field_number,
+  PROTOBUF_ALWAYS_INLINE static uint8* InternalWriteMessageToArray(
-                                                const MessageType& value,
+      int field_number, const MessageType& value, uint8* target);
-                                                uint8* target);
 
   // Like above, but de-virtualize the call to SerializeWithCachedSizes().  The
   // pointer must point at an instance of MessageType, *not* a subclass (or
   // the subclass must not override SerializeWithCachedSizes()).
   template <typename MessageType>
-  INL static uint8* InternalWriteGroupNoVirtualToArray(int field_number,
+  PROTOBUF_ALWAYS_INLINE static uint8* InternalWriteGroupNoVirtualToArray(
-                                                       const MessageType& value,
+      int field_number, const MessageType& value, uint8* target);
-                                                       uint8* target);
   template <typename MessageType>
-  INL static uint8* InternalWriteMessageNoVirtualToArray(
+  PROTOBUF_ALWAYS_INLINE static uint8* InternalWriteMessageNoVirtualToArray(
       int field_number, const MessageType& value, uint8* target);
 
   // For backward-compatibility, the last four methods also have versions
   // that are non-deterministic always.
-  INL static uint8* WriteGroupToArray(int field_number,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteGroupToArray(
-                                      const MessageLite& value, uint8* target) {
+      int field_number, const MessageLite& value, uint8* target) {
     return InternalWriteGroupToArray(field_number, value, target);
   }
-  INL static uint8* WriteMessageToArray(int field_number,
+  PROTOBUF_ALWAYS_INLINE static uint8* WriteMessageToArray(
-                                        const MessageLite& value,
+      int field_number, const MessageLite& value, uint8* target) {
-                                        uint8* target) {
     return InternalWriteMessageToArray(field_number, value, target);
   }
 
-#undef INL
-
   // Compute the byte size of a field.  The XxSize() functions do NOT include
   // the tag, so you must also call TagSize().  (This is because, for repeated
   // fields, you should only call TagSize() once and multiply it by the element

src/google/protobuf/wire_format_lite_inl.h

@@ -1,38 +0,0 @@
-// Protocol Buffers - Google's data interchange format
-// Copyright 2008 Google Inc.  All rights reserved.
-// https://developers.google.com/protocol-buffers/
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-//     * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-//     * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-//     * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef GOOGLE_PROTOBUF_WIRE_FORMAT_LITE_INL_H__
-#define GOOGLE_PROTOBUF_WIRE_FORMAT_LITE_INL_H__
-
-// Please include wire_format_lite.h instead.
-
-#include <google/protobuf/wire_format_lite.h>
-
-#endif  // GOOGLE_PROTOBUF_WIRE_FORMAT_LITE_INL_H__